This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9d97e4a1dfd HIVE-28505: OTEL: Implement OTEL Exporter to expose query
details from HiveServer2 (#5451). (Tanishq Chugh, reviewed by Ayush Saxena)
9d97e4a1dfd is described below
commit 9d97e4a1dfd05a1acca9cb22f7e759cb26151bda
Author: Tanishq Chugh <[email protected]>
AuthorDate: Fri Sep 27 10:27:36 2024 +0530
HIVE-28505: OTEL: Implement OTEL Exporter to expose query details from
HiveServer2 (#5451). (Tanishq Chugh, reviewed by Ayush Saxena)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 +-
pom.xml | 1 +
service/pom.xml | 20 ++
.../apache/hive/service/server/HiveServer2.java | 21 ++
.../apache/hive/service/servlet/OTELExporter.java | 224 +++++++++++++++++++++
5 files changed, 271 insertions(+), 1 deletion(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5252ec8724b..1b181c5ff49 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -5800,7 +5800,11 @@ public class HiveConf extends Configuration {
"tez-site.xml, etc in comma separated list."),
REWRITE_POLICY("hive.rewrite.data.policy", "DEFAULT",
- "Defines the rewrite policy, the valid values are those defined in
RewritePolicy enum");
+ "Defines the rewrite policy, the valid values are those defined in
RewritePolicy enum"),
+
+ HIVE_OTEL_METRICS_FREQUENCY_SECONDS("hive.otel.metrics.frequency.seconds",
"0s",
+ new TimeValidator(TimeUnit.SECONDS),
+ "Frequency at which the OTEL Metrics are refreshed, A value of 0 or
less disable the feature");
public final String varname;
public final String altName;
diff --git a/pom.xml b/pom.xml
index e2157229271..62e29977591 100644
--- a/pom.xml
+++ b/pom.xml
@@ -183,6 +183,7 @@
<oracle.version>21.3.0.0</oracle.version>
<opencsv.version>5.9</opencsv.version>
<orc.version>1.9.4</orc.version>
+ <otel.version>1.42.0</otel.version>
<mockito-core.version>3.4.4</mockito-core.version>
<mockito-inline.version>4.11.0</mockito-inline.version>
<mina.version>2.0.0-M5</mina.version>
diff --git a/service/pom.xml b/service/pom.xml
index dce39eabd07..3bff1ab5ab6 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -365,6 +365,26 @@
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.opentelemetry</groupId>
+ <artifactId>opentelemetry-api</artifactId>
+ <version>${otel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.opentelemetry</groupId>
+ <artifactId>opentelemetry-exporter-logging</artifactId>
+ <version>${otel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.opentelemetry</groupId>
+ <artifactId>opentelemetry-sdk-extension-autoconfigure</artifactId>
+ <version>${otel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.opentelemetry</groupId>
+ <artifactId>opentelemetry-exporter-otlp</artifactId>
+ <version>${otel.version}</version>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>${basedir}/src/java</sourceDirectory>
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java
b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index c410bdd0514..4af82548ac6 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -118,6 +118,7 @@ import org.apache.hive.service.servlet.HS2LeadershipStatus;
import org.apache.hive.service.servlet.HS2Peers;
import org.apache.hive.service.servlet.LDAPAuthenticationFilter;
import org.apache.hive.service.servlet.LoginServlet;
+import org.apache.hive.service.servlet.OTELExporter;
import org.apache.hive.service.servlet.QueriesRESTfulAPIServlet;
import org.apache.hive.service.servlet.QueryProfileServlet;
import org.apache.http.StatusLine;
@@ -134,6 +135,8 @@ import org.apache.zookeeper.ZooDefs.Perms;
import org.apache.zookeeper.data.ACL;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletHolder;
+
+import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -182,6 +185,7 @@ public class HiveServer2 extends CompositeService {
private ZooKeeperHiveHelper zooKeeperHelper = null;
private ScheduledQueryExecutionService scheduledQueryService;
private ServiceContext serviceContext;
+ private OTELExporter otelExporter;
public enum WebUIAuthMethod {
NONE, LDAP
@@ -503,6 +507,19 @@ public class HiveServer2 extends CompositeService {
throw new ServiceException(ie);
}
+ long otelExporterFrequency =
+ hiveConf.getTimeVar(ConfVars.HIVE_OTEL_METRICS_FREQUENCY_SECONDS,
TimeUnit.MILLISECONDS);
+ if (otelExporterFrequency > 0) {
+ otelExporter = new
OTELExporter(AutoConfiguredOpenTelemetrySdk.initialize().getOpenTelemetrySdk(),
+ cliService.getSessionManager(), otelExporterFrequency);
+
+ otelExporter.setName("OTEL Exporter");
+ otelExporter.setDaemon(true);
+ otelExporter.start();
+
+ LOG.info("Started OTEL exporter with frequency {}",
otelExporterFrequency);
+ }
+
// Add a shutdown hook for catching SIGTERM & SIGINT
long timeout = HiveConf.getTimeVar(getHiveConf(),
HiveConf.ConfVars.HIVE_SERVER2_GRACEFUL_STOP_TIMEOUT,
TimeUnit.SECONDS);
@@ -1070,6 +1087,10 @@ public class HiveServer2 extends CompositeService {
HiveSaml2Client.shutdown();
}
+ if (otelExporter != null) {
+ otelExporter.interrupt();
+ }
+
}
private void shutdownExecutor(final ExecutorService
leaderActionsExecutorService) {
diff --git a/service/src/java/org/apache/hive/service/servlet/OTELExporter.java
b/service/src/java/org/apache/hive/service/servlet/OTELExporter.java
new file mode 100644
index 00000000000..8315c4e912e
--- /dev/null
+++ b/service/src/java/org/apache/hive/service/servlet/OTELExporter.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.servlet;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.internal.AttributesMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hive.ql.QueryDisplay;
+import org.apache.hadoop.hive.ql.QueryInfo;
+import org.apache.hive.service.cli.operation.OperationManager;
+import org.apache.hive.service.cli.session.SessionManager;
+
+public class OTELExporter extends Thread {
+ private static final String INSTRUMENTATION_NAME =
OTELExporter.class.getName();
+ private static final Logger LOG =
LoggerFactory.getLogger(OTELExporter.class);
+ private final OperationManager operationManager;
+ private final Set<String> historicalQueryId;
+ private final long frequency;
+ private final Tracer tracer;
+ private final Map<String, Span> queryIdToSpanMap;
+ private final Map<String, Set<String>> queryIdToTasksMap;
+
+ public OTELExporter(OpenTelemetry openTelemetry, SessionManager
sessionManager, long frequency) {
+ this.tracer = openTelemetry.getTracer(INSTRUMENTATION_NAME);
+ this.operationManager = sessionManager.getOperationManager();
+ this.historicalQueryId = new HashSet<>();
+ this.frequency = frequency;
+ this.queryIdToSpanMap = new HashMap<>();
+ this.queryIdToTasksMap = new HashMap<>();
+ }
+
+ @Override
+ public void run() {
+ while (true) {
+ exposeMetricsToOTEL();
+ try {
+ Thread.sleep(frequency);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public void exposeMetricsToOTEL() {
+ List<QueryInfo> liveQueries = operationManager.getLiveQueryInfos();
+ List<QueryInfo> historicalQueries =
operationManager.getHistoricalQueryInfos();
+
+ LOG.debug("Found {} liveQueries and {} historicalQueries",
liveQueries.size(), historicalQueries.size());
+
+ for (QueryInfo lQuery: liveQueries){
+ if(lQuery.getQueryDisplay() == null){
+ continue;
+ }
+ String queryID = lQuery.getQueryDisplay().getQueryId();
+ Span rootspan = queryIdToSpanMap.get(queryID);
+
+ //In case of live query previously encountered in past loops
+ if (rootspan != null) {
+ for (QueryDisplay.TaskDisplay task :
lQuery.getQueryDisplay().getTaskDisplays()) {
+ if (task.getReturnValue() != null && task.getEndTime() != null
+ && queryIdToTasksMap.get(queryID).add(task.getTaskId())) {
+ Context parentContext = Context.current().with(rootspan);
+ tracer.spanBuilder(queryID + " - " + task.getTaskId() + " - live")
+
.setParent(parentContext).setAllAttributes(addTaskAttributes(task))
+ .setStartTimestamp(task.getBeginTime(),
TimeUnit.MILLISECONDS).startSpan()
+ .end(task.getEndTime(), TimeUnit.MILLISECONDS);
+ }
+ }
+ } else {
+ // In case of live queries being seen for first time and has
initialized its queryDisplay
+ rootspan = tracer.spanBuilder(queryID + " - live")
+ .setStartTimestamp(lQuery.getBeginTime(),
TimeUnit.MILLISECONDS).startSpan();
+ Set<String> completedTasks = new HashSet<>();
+ Context parentContext = Context.current().with(rootspan);
+
+ Span initSpan = tracer.spanBuilder(queryID + " -
live").setParent(parentContext)
+ .setStartTimestamp(lQuery.getBeginTime(),
TimeUnit.MILLISECONDS).startSpan()
+ .setAttribute("QueryId", queryID)
+ .setAttribute("QueryString",
lQuery.getQueryDisplay().getQueryString())
+ .setAttribute("UserName", lQuery.getUserName());
+ if (lQuery.getQueryDisplay().getErrorMessage() != null) {
+ initSpan.setAttribute("ErrorMessage",
lQuery.getQueryDisplay().getErrorMessage());
+ }
+ initSpan.end(lQuery.getBeginTime(), TimeUnit.MILLISECONDS);
+
+ for (QueryDisplay.TaskDisplay task :
lQuery.getQueryDisplay().getTaskDisplays()) {
+ if (task.getReturnValue() != null && task.getEndTime() != null) {
+ completedTasks.add(task.getTaskId());
+ parentContext = Context.current().with(rootspan);
+ tracer.spanBuilder(queryID + " - " + task.getTaskId() + " - live")
+
.setParent(parentContext).setAllAttributes(addTaskAttributes(task))
+ .setStartTimestamp(task.getBeginTime(),
TimeUnit.MILLISECONDS).startSpan()
+ .end(task.getEndTime(), TimeUnit.MILLISECONDS);
+ }
+ }
+
+ queryIdToSpanMap.put(queryID, rootspan);
+ queryIdToTasksMap.put(queryID, completedTasks);
+ }
+ }
+
+ Set<String> historicalQueryIDs = new HashSet<>();
+ for (QueryInfo hQuery : historicalQueries) {
+ String hQueryId = hQuery.getQueryDisplay().getQueryId();
+ historicalQueryIDs.add(hQueryId);
+ Span rootspan = queryIdToSpanMap.remove(hQueryId);
+ Set<String> completedTasks = queryIdToTasksMap.remove(hQueryId);
+
+ //For queries that were live till last loop but have ended before start
of this loop
+ if (rootspan != null) {
+ for (QueryDisplay.TaskDisplay task :
hQuery.getQueryDisplay().getTaskDisplays()) {
+ if (!completedTasks.contains(task.getTaskId())) {
+ Context parentContext = Context.current().with(rootspan);
+ tracer.spanBuilder(hQueryId + " - " + task.getTaskId() + " -
completed")
+
.setParent(parentContext).setAllAttributes(addTaskAttributes(task))
+ .setStartTimestamp(task.getBeginTime(),
TimeUnit.MILLISECONDS).startSpan()
+ .end(task.getEndTime(), TimeUnit.MILLISECONDS);
+ }
+ }
+
+ //Update the rootSpan name & attributes before ending it
+ rootspan.updateName(hQueryId + " -
completed").setAllAttributes(addQueryAttributes(hQuery))
+ .end(hQuery.getEndTime(), TimeUnit.MILLISECONDS);
+ historicalQueryId.add(hQueryId);
+ }
+
+ //For queries that already ended either before OTEL service started or
in between OTEL loops
+ if (historicalQueryId.add(hQueryId)) {
+ rootspan = tracer.spanBuilder(hQueryId + " - completed")
+ .setStartTimestamp(hQuery.getBeginTime(),
TimeUnit.MILLISECONDS).startSpan();
+ Context parentContext = Context.current().with(rootspan);
+
+ Span initSpan = tracer.spanBuilder(hQueryId + " -
completed").setParent(parentContext)
+ .setStartTimestamp(hQuery.getBeginTime(),
TimeUnit.MILLISECONDS).startSpan()
+ .setAttribute("QueryId", hQueryId)
+ .setAttribute("QueryString",
hQuery.getQueryDisplay().getQueryString())
+ .setAttribute("UserName", hQuery.getUserName());
+ if (hQuery.getQueryDisplay().getErrorMessage() != null) {
+ initSpan.setAttribute("ErrorMessage",
hQuery.getQueryDisplay().getErrorMessage());
+ }
+ initSpan.end(hQuery.getBeginTime(), TimeUnit.MILLISECONDS);
+
+ for (QueryDisplay.TaskDisplay task :
hQuery.getQueryDisplay().getTaskDisplays()) {
+ parentContext = Context.current().with(rootspan);
+ tracer.spanBuilder(hQueryId + " - " + task.getTaskId() + " -
completed")
+
.setParent(parentContext).setAllAttributes(addTaskAttributes(task))
+ .setStartTimestamp(task.getBeginTime(),
TimeUnit.MILLISECONDS).startSpan()
+ .end(task.getEndTime(), TimeUnit.MILLISECONDS);
+ }
+
+
rootspan.setAllAttributes(addQueryAttributes(hQuery)).end(hQuery.getEndTime(),
TimeUnit.MILLISECONDS);
+ }
+ }
+
+ historicalQueryId.retainAll(historicalQueryIDs);
+ }
+
+ private AttributesMap addQueryAttributes(QueryInfo query){
+ AttributesMap attributes = AttributesMap.create(Long.MAX_VALUE,
Integer.MAX_VALUE);
+ attributes.put(AttributeKey.stringKey("QueryId"),
query.getQueryDisplay().getQueryId());
+ attributes.put(AttributeKey.longKey("QueryStartTime"),
query.getQueryDisplay().getQueryStartTime());
+ attributes.put(AttributeKey.longKey("EndTime"), query.getEndTime());
+ attributes.put(AttributeKey.stringKey("OperationId"),
query.getOperationId());
+ attributes.put(AttributeKey.stringKey("OperationLogLocation"),
query.getOperationLogLocation());
+ attributes.put(AttributeKey.stringKey("ErrorMessage"),
query.getQueryDisplay().getErrorMessage());
+ attributes.put(AttributeKey.stringKey("ExplainPlan"),
query.getQueryDisplay().getExplainPlan());
+ attributes.put(AttributeKey.stringKey("FullLogLocation"),
query.getQueryDisplay().getFullLogLocation());
+ attributes.put(AttributeKey.stringKey("Running"),
String.valueOf(query.isRunning()));
+ attributes.put(AttributeKey.longKey("Runtime"), query.getRuntime());
+ attributes.put(AttributeKey.stringKey("UserName"), query.getUserName());
+ attributes.put(AttributeKey.stringKey("State"), query.getState());
+ attributes.put(AttributeKey.stringKey("SessionId"), query.getSessionId());
+ return attributes;
+ }
+
+ private AttributesMap addTaskAttributes(QueryDisplay.TaskDisplay
taskDisplay) {
+ AttributesMap attributes = AttributesMap.create(Long.MAX_VALUE,
Integer.MAX_VALUE);
+ attributes.put(AttributeKey.stringKey("TaskId"), taskDisplay.getTaskId());
+ attributes.put(AttributeKey.stringKey("Name"), taskDisplay.getName());
+ if(taskDisplay.getTaskType() != null){
+ attributes.put(AttributeKey.stringKey("TaskType"),
taskDisplay.getTaskType().toString());
+ }
+ attributes.put(AttributeKey.stringKey("Status"), taskDisplay.getStatus());
+ attributes.put(AttributeKey.stringKey("StatusMessage"),
taskDisplay.getStatusMessage());
+ attributes.put(AttributeKey.stringKey("ExternalHandle"),
taskDisplay.getExternalHandle());
+ attributes.put(AttributeKey.stringKey("ErrorMsg"),
taskDisplay.getErrorMsg());
+ if(taskDisplay.getReturnValue() != null ){
+ attributes.put(AttributeKey.longKey("ReturnValue"),
taskDisplay.getReturnValue().longValue());
+ }
+ attributes.put(AttributeKey.longKey("BeginTime"),
taskDisplay.getBeginTime());
+ attributes.put(AttributeKey.longKey("ElapsedTime"),
taskDisplay.getElapsedTime());
+ attributes.put(AttributeKey.longKey("EndTime"), taskDisplay.getEndTime());
+ return attributes;
+ }
+}