This is an automated email from the ASF dual-hosted git repository.

dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new b552630ca0b HIVE-28555: Add custom plugins support to ReExecution (Ryu 
Kobayashi, reviewed by Denys Kuzmenko)
b552630ca0b is described below

commit b552630ca0bab60bdd858879f5cd4778e77b5ddd
Author: Ryu Kobayashi <[email protected]>
AuthorDate: Wed Oct 16 01:59:14 2024 +0900

    HIVE-28555: Add custom plugins support to ReExecution (Ryu Kobayashi, 
reviewed by Denys Kuzmenko)
    
    Closes #5487
---
 .../java/org/apache/hadoop/hive/conf/HiveConf.java |   6 +-
 .../org/apache/hadoop/hive/ql/DriverFactory.java   |  26 +++-
 .../apache/hadoop/hive/ql/reexec/ReExecDriver.java |   5 +
 .../hive/ql/reexec/ReExecutionStrategyType.java    |   2 -
 .../apache/hadoop/hive/ql/TestDriverFactory.java   | 141 +++++++++++++++++++++
 5 files changed, 172 insertions(+), 8 deletions(-)

diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index b6bb8ea3d55..84a227d6c06 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -5681,12 +5681,14 @@ public class HiveConf extends Configuration {
         "Enable query reexecutions"),
     HIVE_QUERY_REEXECUTION_STRATEGIES("hive.query.reexecution.strategies",
         
"overlay,reoptimize,reexecute_lost_am,dagsubmit,recompile_without_cbo,write_conflict",
-        "comma separated list of plugin can be used:\n"
+        "comma separated list of plugin can be used. If custom plugins, 
specify the class name:\n"
             + "  overlay: hiveconf subtree 'reexec.overlay' is used as an 
overlay in case of an execution errors out\n"
             + "  reoptimize: collects operator statistics during execution and 
recompile the query after a failure\n"
             + "  recompile_without_cbo: recompiles query after a CBO failure\n"
             + "  reexecute_lost_am: reexecutes query if it failed due to tez 
am node gets decommissioned\n "
-            + "  write_conflict: retries the query once if the query failed 
due to write_conflict"),
+            + "  write_conflict: retries the query once if the query failed 
due to write_conflict\n"
+            + "  custom plugins: e.g.\n"
+            + "    org.apache.hadoop.hive.ql.reexec.custom.CustomPlugin1"),
     
HIVE_QUERY_REEXECUTION_STATS_PERSISTENCE("hive.query.reexecution.stats.persist.scope",
 "metastore",
         new StringSet("query", "hiveserver", "metastore"),
         "Sets the persistence scope of runtime statistics\n"
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java 
b/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java
index 25357da6566..2e29cf3206a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql;
 
+import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -49,12 +50,13 @@ public final class DriverFactory {
     }
 
     String strategies = 
queryState.getConf().getVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES);
-    strategies = Strings.nullToEmpty(strategies).trim().toLowerCase();
+    strategies = Strings.nullToEmpty(strategies);
     List<IReExecutionPlugin> plugins = new ArrayList<>();
     for (String string : strategies.split(",")) {
       if (string.trim().isEmpty()) {
         continue;
       }
+
       plugins.add(buildReExecPlugin(string));
     }
 
@@ -62,10 +64,26 @@ public final class DriverFactory {
   }
 
   private static IReExecutionPlugin buildReExecPlugin(String name) throws 
RuntimeException {
-    Class<? extends IReExecutionPlugin> pluginType = 
ReExecutionStrategyType.getPluginClassByName(name);
+    Class<? extends IReExecutionPlugin> pluginType;
+    try {
+      pluginType = ReExecutionStrategyType.getPluginClassByName(name);
+    } catch (IllegalArgumentException e) {
+        try {
+          Class<?> cls = Class.forName(name);
+          if (cls.isAssignableFrom(IReExecutionPlugin.class)) {
+            throw new RuntimeException("Not re-execution plugin: " + name);
+          }
+
+          pluginType = (Class<? extends IReExecutionPlugin>) cls;
+        } catch (ClassNotFoundException e1) {
+          throw new RuntimeException(
+              "Unknown re-execution plugin: " + name + " (" + 
ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES.varname + ")");
+        }
+    }
+
     try {
-      return pluginType.newInstance();
-    } catch (InstantiationException | IllegalAccessException e) {
+      return pluginType.getDeclaredConstructor(null).newInstance(null);
+    } catch (InstantiationException | IllegalAccessException | 
InvocationTargetException | NoSuchMethodException e) {
       throw new RuntimeException(
           "Unknown re-execution plugin: " + name + " (" + 
ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES.varname + ")");
     }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
index 94ef3932aa8..48a38e9f416 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
@@ -83,6 +83,11 @@ public class ReExecDriver implements IDriver {
     return coreDriver.compile(command, resetTaskIds);
   }
 
+  @VisibleForTesting
+  public List<IReExecutionPlugin> getPlugins() {
+    return plugins;
+  }
+
   private boolean firstExecution() {
     return executionIndex == 0;
   }
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionStrategyType.java 
b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionStrategyType.java
index 9f50acc71ed..17046c899db 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionStrategyType.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionStrategyType.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.hive.ql.reexec;
 
-import java.util.Optional;
-
 public enum ReExecutionStrategyType {
   OVERLAY(ReExecutionOverlayPlugin.class),
   REOPTIMIZE(ReOptimizePlugin.class),
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestDriverFactory.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestDriverFactory.java
new file mode 100644
index 00000000000..524441f5fc5
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestDriverFactory.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import static org.junit.Assert.fail;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.reexec.IReExecutionPlugin;
+import org.apache.hadoop.hive.ql.reexec.ReExecDriver;
+import org.apache.hadoop.hive.ql.reexec.ReExecutionStrategyType;
+import org.junit.Test;
+
+public class TestDriverFactory {
+
+  @Test
+  public void testNormal() {
+    HiveConf conf = new HiveConf();
+    IDriver driver = DriverFactory.newDriver(conf);
+    ReExecDriver reDriver = (ReExecDriver) driver;
+
+    List<IReExecutionPlugin> plugins = getPlugins(conf);
+    for (IReExecutionPlugin original : plugins) {
+      boolean found = false;
+      for (IReExecutionPlugin instance : reDriver.getPlugins()) {
+        if 
(original.getClass().getName().equals(instance.getClass().getName())) {
+          found = true;
+        }
+      }
+
+      if (!found) {
+        fail("The ReExecutionPlugin defined has not been instantiated");
+      }
+    }
+  }
+
+  @Test(expected = RuntimeException.class)
+  public void testNormalFailed() {
+    HiveConf conf = new HiveConf();
+    conf.setVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES,
+        "overlay,reoptimize,reexecute_lost_am,dagsubmit,test");
+
+    DriverFactory.newDriver(conf);
+  }
+
+  @Test
+  public void testNormalAndCustom() {
+    HiveConf conf = new HiveConf();
+    conf.setVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES,
+        "overlay,reoptimize,reexecute_lost_am,dagsubmit" + 
+        ",org.apache.hadoop.hive.ql.reexec.ReCompileWithoutCBOPlugin" +
+        ",org.apache.hadoop.hive.ql.reexec.ReExecuteOnWriteConflictPlugin");
+
+    IDriver driver = DriverFactory.newDriver(conf);
+    ReExecDriver reDriver = (ReExecDriver) driver;
+
+    List<IReExecutionPlugin> plugins = getPlugins(conf);
+    for (IReExecutionPlugin original : plugins) {
+      boolean found = false;
+      for (IReExecutionPlugin instance : reDriver.getPlugins()) {
+        if 
(original.getClass().getName().equals(instance.getClass().getName())) {
+          found = true;
+        }
+      }
+
+      if (!found) {
+        fail("The ReExecutionPlugin defined has not been instantiated");
+      }
+    }
+  }
+
+  @Test(expected = RuntimeException.class)
+  public void testCustomNotInstanceOfIReExecutionPlugin() {
+    HiveConf conf = new HiveConf();
+    conf.setVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES,
+        "overlay,reoptimize,reexecute_lost_am,dagsubmit" +
+        ",org.apache.hadoop.hive.conf.HiveConf");
+
+    DriverFactory.newDriver(conf);
+  }
+
+  private List<IReExecutionPlugin> getPlugins(HiveConf conf) {
+    String strategies = 
conf.getVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES);
+    List<IReExecutionPlugin> plugins = new ArrayList<>();
+    for (String string : strategies.split(",")) {
+      if (string.trim().isEmpty()) {
+        continue;
+      }
+
+      plugins.add(buildReExecPlugin(string));
+    }
+
+    return plugins;
+  }
+
+  private static IReExecutionPlugin buildReExecPlugin(String name) throws 
RuntimeException {
+    Class<? extends IReExecutionPlugin> pluginType;
+    try {
+      pluginType = ReExecutionStrategyType.getPluginClassByName(name);
+    } catch (IllegalArgumentException e) {
+        try {
+          Class<?> cls = Class.forName(name);
+          if (cls.isAssignableFrom(IReExecutionPlugin.class)) {
+            throw new RuntimeException("Not re-execution plugin: " + name);
+          }
+
+          pluginType = (Class<? extends IReExecutionPlugin>) cls;
+        } catch (ClassNotFoundException e1) {
+          throw new RuntimeException(
+              "Unknown re-execution plugin: " + name + " (" + 
ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES.varname + ")");
+        }
+    }
+
+    try {
+      return pluginType.getDeclaredConstructor(null).newInstance(null);
+    } catch (InstantiationException | IllegalAccessException | 
InvocationTargetException | NoSuchMethodException e) {
+      throw new RuntimeException(
+          "Unknown re-execution plugin: " + name + " (" + 
ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES.varname + ")");
+    }
+  }
+}

Reply via email to