This is an automated email from the ASF dual-hosted git repository.

ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 9d97e4a1dfd HIVE-28505: OTEL: Implement OTEL Exporter to expose query 
details from HiveServer2 (#5451). (Tanishq Chugh, reviewed by Ayush Saxena)
9d97e4a1dfd is described below

commit 9d97e4a1dfd05a1acca9cb22f7e759cb26151bda
Author: Tanishq Chugh <[email protected]>
AuthorDate: Fri Sep 27 10:27:36 2024 +0530

    HIVE-28505: OTEL: Implement OTEL Exporter to expose query details from 
HiveServer2 (#5451). (Tanishq Chugh, reviewed by Ayush Saxena)
---
 .../java/org/apache/hadoop/hive/conf/HiveConf.java |   6 +-
 pom.xml                                            |   1 +
 service/pom.xml                                    |  20 ++
 .../apache/hive/service/server/HiveServer2.java    |  21 ++
 .../apache/hive/service/servlet/OTELExporter.java  | 224 +++++++++++++++++++++
 5 files changed, 271 insertions(+), 1 deletion(-)

diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5252ec8724b..1b181c5ff49 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -5800,7 +5800,11 @@ public class HiveConf extends Configuration {
                     "tez-site.xml, etc in comma separated list."),
 
     REWRITE_POLICY("hive.rewrite.data.policy", "DEFAULT", 
-        "Defines the rewrite policy, the valid values are those defined in 
RewritePolicy enum"); 
+        "Defines the rewrite policy, the valid values are those defined in 
RewritePolicy enum"),
+
+    HIVE_OTEL_METRICS_FREQUENCY_SECONDS("hive.otel.metrics.frequency.seconds", 
"0s",
+        new TimeValidator(TimeUnit.SECONDS),
+        "Frequency at which the OTEL Metrics are refreshed, A value of 0 or 
less disable the feature");
 
     public final String varname;
     public final String altName;
diff --git a/pom.xml b/pom.xml
index e2157229271..62e29977591 100644
--- a/pom.xml
+++ b/pom.xml
@@ -183,6 +183,7 @@
     <oracle.version>21.3.0.0</oracle.version>
     <opencsv.version>5.9</opencsv.version>
     <orc.version>1.9.4</orc.version>
+    <otel.version>1.42.0</otel.version>
     <mockito-core.version>3.4.4</mockito-core.version>
     <mockito-inline.version>4.11.0</mockito-inline.version>
     <mina.version>2.0.0-M5</mina.version>
diff --git a/service/pom.xml b/service/pom.xml
index dce39eabd07..3bff1ab5ab6 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -365,6 +365,26 @@
       <version>${hadoop.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-api</artifactId>
+      <version>${otel.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-exporter-logging</artifactId>
+      <version>${otel.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-extension-autoconfigure</artifactId>
+      <version>${otel.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-exporter-otlp</artifactId>
+      <version>${otel.version}</version>
+    </dependency>
   </dependencies>
   <build>
     <sourceDirectory>${basedir}/src/java</sourceDirectory>
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java 
b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index c410bdd0514..4af82548ac6 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -118,6 +118,7 @@ import org.apache.hive.service.servlet.HS2LeadershipStatus;
 import org.apache.hive.service.servlet.HS2Peers;
 import org.apache.hive.service.servlet.LDAPAuthenticationFilter;
 import org.apache.hive.service.servlet.LoginServlet;
+import org.apache.hive.service.servlet.OTELExporter;
 import org.apache.hive.service.servlet.QueriesRESTfulAPIServlet;
 import org.apache.hive.service.servlet.QueryProfileServlet;
 import org.apache.http.StatusLine;
@@ -134,6 +135,8 @@ import org.apache.zookeeper.ZooDefs.Perms;
 import org.apache.zookeeper.data.ACL;
 import org.eclipse.jetty.servlet.FilterHolder;
 import org.eclipse.jetty.servlet.ServletHolder;
+
+import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -182,6 +185,7 @@ public class HiveServer2 extends CompositeService {
   private ZooKeeperHiveHelper zooKeeperHelper = null;
   private ScheduledQueryExecutionService scheduledQueryService;
   private ServiceContext serviceContext;
+  private OTELExporter otelExporter;
 
   public enum WebUIAuthMethod {
     NONE, LDAP
@@ -503,6 +507,19 @@ public class HiveServer2 extends CompositeService {
       throw new ServiceException(ie);
     }
 
+    long otelExporterFrequency =
+        hiveConf.getTimeVar(ConfVars.HIVE_OTEL_METRICS_FREQUENCY_SECONDS, 
TimeUnit.MILLISECONDS);
+    if (otelExporterFrequency > 0) {
+      otelExporter = new 
OTELExporter(AutoConfiguredOpenTelemetrySdk.initialize().getOpenTelemetrySdk(),
+          cliService.getSessionManager(), otelExporterFrequency);
+
+      otelExporter.setName("OTEL Exporter");
+      otelExporter.setDaemon(true);
+      otelExporter.start();
+
+      LOG.info("Started OTEL exporter with frequency {}", 
otelExporterFrequency);
+    }
+
     // Add a shutdown hook for catching SIGTERM & SIGINT
     long timeout = HiveConf.getTimeVar(getHiveConf(),
         HiveConf.ConfVars.HIVE_SERVER2_GRACEFUL_STOP_TIMEOUT, 
TimeUnit.SECONDS);
@@ -1070,6 +1087,10 @@ public class HiveServer2 extends CompositeService {
       HiveSaml2Client.shutdown();
     }
 
+    if (otelExporter != null) {
+      otelExporter.interrupt();
+    }
+
   }
 
   private void shutdownExecutor(final ExecutorService 
leaderActionsExecutorService) {
diff --git a/service/src/java/org/apache/hive/service/servlet/OTELExporter.java 
b/service/src/java/org/apache/hive/service/servlet/OTELExporter.java
new file mode 100644
index 00000000000..8315c4e912e
--- /dev/null
+++ b/service/src/java/org/apache/hive/service/servlet/OTELExporter.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.servlet;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.internal.AttributesMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hive.ql.QueryDisplay;
+import org.apache.hadoop.hive.ql.QueryInfo;
+import org.apache.hive.service.cli.operation.OperationManager;
+import org.apache.hive.service.cli.session.SessionManager;
+
+public class OTELExporter extends Thread {
+  private static final String INSTRUMENTATION_NAME = 
OTELExporter.class.getName();
+  private static final Logger LOG = 
LoggerFactory.getLogger(OTELExporter.class);
+  private final OperationManager operationManager;
+  private final Set<String> historicalQueryId;
+  private final long frequency;
+  private final Tracer tracer;
+  private final Map<String, Span> queryIdToSpanMap;
+  private final Map<String, Set<String>> queryIdToTasksMap;
+
+  public OTELExporter(OpenTelemetry openTelemetry, SessionManager 
sessionManager, long frequency) {
+    this.tracer = openTelemetry.getTracer(INSTRUMENTATION_NAME);
+    this.operationManager = sessionManager.getOperationManager();
+    this.historicalQueryId = new HashSet<>();
+    this.frequency = frequency;
+    this.queryIdToSpanMap = new HashMap<>();
+    this.queryIdToTasksMap = new HashMap<>();
+  }
+
+  @Override
+  public void run() {
+    while (true) {
+      exposeMetricsToOTEL();
+      try {
+        Thread.sleep(frequency);
+      } catch (InterruptedException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  public void exposeMetricsToOTEL() {
+    List<QueryInfo> liveQueries = operationManager.getLiveQueryInfos();
+    List<QueryInfo> historicalQueries = 
operationManager.getHistoricalQueryInfos();
+
+    LOG.debug("Found {} liveQueries and {} historicalQueries", 
liveQueries.size(), historicalQueries.size());
+
+    for (QueryInfo lQuery: liveQueries){
+      if(lQuery.getQueryDisplay() == null){
+        continue;
+      }
+      String queryID = lQuery.getQueryDisplay().getQueryId();
+      Span rootspan = queryIdToSpanMap.get(queryID);
+
+      //In case of live query previously encountered in past loops
+      if (rootspan != null) {
+        for (QueryDisplay.TaskDisplay task : 
lQuery.getQueryDisplay().getTaskDisplays()) {
+          if (task.getReturnValue() != null && task.getEndTime() != null
+                  && queryIdToTasksMap.get(queryID).add(task.getTaskId())) {
+            Context parentContext = Context.current().with(rootspan);
+            tracer.spanBuilder(queryID + " - " + task.getTaskId() + " - live")
+                    
.setParent(parentContext).setAllAttributes(addTaskAttributes(task))
+                    .setStartTimestamp(task.getBeginTime(), 
TimeUnit.MILLISECONDS).startSpan()
+                    .end(task.getEndTime(), TimeUnit.MILLISECONDS);
+          }
+        }
+      } else {
+        // In case of live queries being seen for first time and has 
initialized its queryDisplay
+        rootspan = tracer.spanBuilder(queryID + " - live")
+                .setStartTimestamp(lQuery.getBeginTime(), 
TimeUnit.MILLISECONDS).startSpan();
+        Set<String> completedTasks = new HashSet<>();
+        Context parentContext = Context.current().with(rootspan);
+
+        Span initSpan = tracer.spanBuilder(queryID + " - 
live").setParent(parentContext)
+                .setStartTimestamp(lQuery.getBeginTime(), 
TimeUnit.MILLISECONDS).startSpan()
+                .setAttribute("QueryId", queryID)
+                .setAttribute("QueryString", 
lQuery.getQueryDisplay().getQueryString())
+                .setAttribute("UserName", lQuery.getUserName());
+        if (lQuery.getQueryDisplay().getErrorMessage() != null) {
+          initSpan.setAttribute("ErrorMessage", 
lQuery.getQueryDisplay().getErrorMessage());
+        }
+        initSpan.end(lQuery.getBeginTime(), TimeUnit.MILLISECONDS);
+
+        for (QueryDisplay.TaskDisplay task : 
lQuery.getQueryDisplay().getTaskDisplays()) {
+          if (task.getReturnValue() != null && task.getEndTime() != null) {
+            completedTasks.add(task.getTaskId());
+            parentContext = Context.current().with(rootspan);
+            tracer.spanBuilder(queryID + " - " + task.getTaskId() + " - live")
+                    
.setParent(parentContext).setAllAttributes(addTaskAttributes(task))
+                    .setStartTimestamp(task.getBeginTime(), 
TimeUnit.MILLISECONDS).startSpan()
+                    .end(task.getEndTime(), TimeUnit.MILLISECONDS);
+          }
+        }
+        
+        queryIdToSpanMap.put(queryID, rootspan);
+        queryIdToTasksMap.put(queryID, completedTasks);
+      }
+    }
+
+    Set<String> historicalQueryIDs = new HashSet<>();
+    for (QueryInfo hQuery : historicalQueries) {
+      String hQueryId = hQuery.getQueryDisplay().getQueryId();
+      historicalQueryIDs.add(hQueryId);
+      Span rootspan = queryIdToSpanMap.remove(hQueryId);
+      Set<String> completedTasks = queryIdToTasksMap.remove(hQueryId);
+
+      //For queries that were live till last loop but have ended before start 
of this loop
+      if (rootspan != null) {
+        for (QueryDisplay.TaskDisplay task : 
hQuery.getQueryDisplay().getTaskDisplays()) {
+          if (!completedTasks.contains(task.getTaskId())) {
+            Context parentContext = Context.current().with(rootspan);
+            tracer.spanBuilder(hQueryId + " - " + task.getTaskId() + " - 
completed")
+                    
.setParent(parentContext).setAllAttributes(addTaskAttributes(task))
+                    .setStartTimestamp(task.getBeginTime(), 
TimeUnit.MILLISECONDS).startSpan()
+                    .end(task.getEndTime(), TimeUnit.MILLISECONDS);
+          }
+        }
+
+        //Update the rootSpan name & attributes before ending it
+        rootspan.updateName(hQueryId + " - 
completed").setAllAttributes(addQueryAttributes(hQuery))
+                .end(hQuery.getEndTime(), TimeUnit.MILLISECONDS);
+        historicalQueryId.add(hQueryId);
+      }
+
+      //For queries that already ended either before OTEL service started or 
in between OTEL loops
+      if (historicalQueryId.add(hQueryId)) {
+        rootspan = tracer.spanBuilder(hQueryId + " - completed")
+                .setStartTimestamp(hQuery.getBeginTime(), 
TimeUnit.MILLISECONDS).startSpan();
+        Context parentContext = Context.current().with(rootspan);
+
+        Span initSpan = tracer.spanBuilder(hQueryId + " - 
completed").setParent(parentContext)
+                .setStartTimestamp(hQuery.getBeginTime(), 
TimeUnit.MILLISECONDS).startSpan()
+                .setAttribute("QueryId", hQueryId)
+                .setAttribute("QueryString", 
hQuery.getQueryDisplay().getQueryString())
+                .setAttribute("UserName", hQuery.getUserName());
+        if (hQuery.getQueryDisplay().getErrorMessage() != null) {
+          initSpan.setAttribute("ErrorMessage", 
hQuery.getQueryDisplay().getErrorMessage());
+        }
+        initSpan.end(hQuery.getBeginTime(), TimeUnit.MILLISECONDS);
+
+        for (QueryDisplay.TaskDisplay task : 
hQuery.getQueryDisplay().getTaskDisplays()) {
+          parentContext = Context.current().with(rootspan);
+          tracer.spanBuilder(hQueryId + " - " + task.getTaskId() + " - 
completed")
+                  
.setParent(parentContext).setAllAttributes(addTaskAttributes(task))
+                  .setStartTimestamp(task.getBeginTime(), 
TimeUnit.MILLISECONDS).startSpan()
+                  .end(task.getEndTime(), TimeUnit.MILLISECONDS);
+        }
+        
+        
rootspan.setAllAttributes(addQueryAttributes(hQuery)).end(hQuery.getEndTime(), 
TimeUnit.MILLISECONDS);
+      }
+    }
+    
+    historicalQueryId.retainAll(historicalQueryIDs);
+  }
+
+  private AttributesMap addQueryAttributes(QueryInfo query){
+    AttributesMap attributes = AttributesMap.create(Long.MAX_VALUE, 
Integer.MAX_VALUE);
+    attributes.put(AttributeKey.stringKey("QueryId"), 
query.getQueryDisplay().getQueryId());
+    attributes.put(AttributeKey.longKey("QueryStartTime"), 
query.getQueryDisplay().getQueryStartTime());
+    attributes.put(AttributeKey.longKey("EndTime"), query.getEndTime());
+    attributes.put(AttributeKey.stringKey("OperationId"), 
query.getOperationId());
+    attributes.put(AttributeKey.stringKey("OperationLogLocation"), 
query.getOperationLogLocation());
+    attributes.put(AttributeKey.stringKey("ErrorMessage"), 
query.getQueryDisplay().getErrorMessage());
+    attributes.put(AttributeKey.stringKey("ExplainPlan"), 
query.getQueryDisplay().getExplainPlan());
+    attributes.put(AttributeKey.stringKey("FullLogLocation"), 
query.getQueryDisplay().getFullLogLocation());
+    attributes.put(AttributeKey.stringKey("Running"), 
String.valueOf(query.isRunning()));
+    attributes.put(AttributeKey.longKey("Runtime"), query.getRuntime());
+    attributes.put(AttributeKey.stringKey("UserName"), query.getUserName());
+    attributes.put(AttributeKey.stringKey("State"), query.getState());
+    attributes.put(AttributeKey.stringKey("SessionId"), query.getSessionId());
+    return attributes;
+  }
+
+  private AttributesMap addTaskAttributes(QueryDisplay.TaskDisplay 
taskDisplay) {
+    AttributesMap attributes = AttributesMap.create(Long.MAX_VALUE, 
Integer.MAX_VALUE);
+    attributes.put(AttributeKey.stringKey("TaskId"), taskDisplay.getTaskId());
+    attributes.put(AttributeKey.stringKey("Name"), taskDisplay.getName());
+    if(taskDisplay.getTaskType() != null){
+      attributes.put(AttributeKey.stringKey("TaskType"), 
taskDisplay.getTaskType().toString());
+    }
+    attributes.put(AttributeKey.stringKey("Status"), taskDisplay.getStatus());
+    attributes.put(AttributeKey.stringKey("StatusMessage"), 
taskDisplay.getStatusMessage());
+    attributes.put(AttributeKey.stringKey("ExternalHandle"), 
taskDisplay.getExternalHandle());
+    attributes.put(AttributeKey.stringKey("ErrorMsg"), 
taskDisplay.getErrorMsg());
+    if(taskDisplay.getReturnValue() != null ){
+      attributes.put(AttributeKey.longKey("ReturnValue"), 
taskDisplay.getReturnValue().longValue());
+    }
+    attributes.put(AttributeKey.longKey("BeginTime"), 
taskDisplay.getBeginTime());
+    attributes.put(AttributeKey.longKey("ElapsedTime"), 
taskDisplay.getElapsedTime());
+    attributes.put(AttributeKey.longKey("EndTime"), taskDisplay.getEndTime());
+    return attributes;
+  }
+}

Reply via email to