This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b552630ca0b HIVE-28555: Add custom plugins support to ReExecution (Ryu
Kobayashi, reviewed by Denys Kuzmenko)
b552630ca0b is described below
commit b552630ca0bab60bdd858879f5cd4778e77b5ddd
Author: Ryu Kobayashi <[email protected]>
AuthorDate: Wed Oct 16 01:59:14 2024 +0900
HIVE-28555: Add custom plugins support to ReExecution (Ryu Kobayashi,
reviewed by Denys Kuzmenko)
Closes #5487
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 +-
.../org/apache/hadoop/hive/ql/DriverFactory.java | 26 +++-
.../apache/hadoop/hive/ql/reexec/ReExecDriver.java | 5 +
.../hive/ql/reexec/ReExecutionStrategyType.java | 2 -
.../apache/hadoop/hive/ql/TestDriverFactory.java | 141 +++++++++++++++++++++
5 files changed, 172 insertions(+), 8 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index b6bb8ea3d55..84a227d6c06 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -5681,12 +5681,14 @@ public class HiveConf extends Configuration {
"Enable query reexecutions"),
HIVE_QUERY_REEXECUTION_STRATEGIES("hive.query.reexecution.strategies",
"overlay,reoptimize,reexecute_lost_am,dagsubmit,recompile_without_cbo,write_conflict",
- "comma separated list of plugin can be used:\n"
+ "comma separated list of plugin can be used. If custom plugins,
specify the class name:\n"
+ " overlay: hiveconf subtree 'reexec.overlay' is used as an
overlay in case of an execution errors out\n"
+ " reoptimize: collects operator statistics during execution and
recompile the query after a failure\n"
+ " recompile_without_cbo: recompiles query after a CBO failure\n"
+ " reexecute_lost_am: reexecutes query if it failed due to tez
am node gets decommissioned\n "
- + " write_conflict: retries the query once if the query failed
due to write_conflict"),
+ + " write_conflict: retries the query once if the query failed
due to write_conflict\n"
+ + " custom plugins: e.g.\n"
+ + " org.apache.hadoop.hive.ql.reexec.custom.CustomPlugin1"),
HIVE_QUERY_REEXECUTION_STATS_PERSISTENCE("hive.query.reexecution.stats.persist.scope",
"metastore",
new StringSet("query", "hiveserver", "metastore"),
"Sets the persistence scope of runtime statistics\n"
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java
index 25357da6566..2e29cf3206a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
@@ -49,12 +50,13 @@ public final class DriverFactory {
}
String strategies =
queryState.getConf().getVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES);
- strategies = Strings.nullToEmpty(strategies).trim().toLowerCase();
+ strategies = Strings.nullToEmpty(strategies);
List<IReExecutionPlugin> plugins = new ArrayList<>();
for (String string : strategies.split(",")) {
if (string.trim().isEmpty()) {
continue;
}
+
plugins.add(buildReExecPlugin(string));
}
@@ -62,10 +64,26 @@ public final class DriverFactory {
}
private static IReExecutionPlugin buildReExecPlugin(String name) throws
RuntimeException {
- Class<? extends IReExecutionPlugin> pluginType =
ReExecutionStrategyType.getPluginClassByName(name);
+ Class<? extends IReExecutionPlugin> pluginType;
+ try {
+ pluginType = ReExecutionStrategyType.getPluginClassByName(name);
+ } catch (IllegalArgumentException e) {
+ try {
+ Class<?> cls = Class.forName(name);
+ if (cls.isAssignableFrom(IReExecutionPlugin.class)) {
+ throw new RuntimeException("Not re-execution plugin: " + name);
+ }
+
+ pluginType = (Class<? extends IReExecutionPlugin>) cls;
+ } catch (ClassNotFoundException e1) {
+ throw new RuntimeException(
+ "Unknown re-execution plugin: " + name + " (" +
ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES.varname + ")");
+ }
+ }
+
try {
- return pluginType.newInstance();
- } catch (InstantiationException | IllegalAccessException e) {
+ return pluginType.getDeclaredConstructor(null).newInstance(null);
+ } catch (InstantiationException | IllegalAccessException |
InvocationTargetException | NoSuchMethodException e) {
throw new RuntimeException(
"Unknown re-execution plugin: " + name + " (" +
ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES.varname + ")");
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
index 94ef3932aa8..48a38e9f416 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
@@ -83,6 +83,11 @@ public class ReExecDriver implements IDriver {
return coreDriver.compile(command, resetTaskIds);
}
+ @VisibleForTesting
+ public List<IReExecutionPlugin> getPlugins() {
+ return plugins;
+ }
+
private boolean firstExecution() {
return executionIndex == 0;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionStrategyType.java
b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionStrategyType.java
index 9f50acc71ed..17046c899db 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionStrategyType.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionStrategyType.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.hive.ql.reexec;
-import java.util.Optional;
-
public enum ReExecutionStrategyType {
OVERLAY(ReExecutionOverlayPlugin.class),
REOPTIMIZE(ReOptimizePlugin.class),
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestDriverFactory.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestDriverFactory.java
new file mode 100644
index 00000000000..524441f5fc5
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestDriverFactory.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import static org.junit.Assert.fail;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.reexec.IReExecutionPlugin;
+import org.apache.hadoop.hive.ql.reexec.ReExecDriver;
+import org.apache.hadoop.hive.ql.reexec.ReExecutionStrategyType;
+import org.junit.Test;
+
+public class TestDriverFactory {
+
+ @Test
+ public void testNormal() {
+ HiveConf conf = new HiveConf();
+ IDriver driver = DriverFactory.newDriver(conf);
+ ReExecDriver reDriver = (ReExecDriver) driver;
+
+ List<IReExecutionPlugin> plugins = getPlugins(conf);
+ for (IReExecutionPlugin original : plugins) {
+ boolean found = false;
+ for (IReExecutionPlugin instance : reDriver.getPlugins()) {
+ if
(original.getClass().getName().equals(instance.getClass().getName())) {
+ found = true;
+ }
+ }
+
+ if (!found) {
+ fail("The ReExecutionPlugin defined has not been instantiated");
+ }
+ }
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testNormalFailed() {
+ HiveConf conf = new HiveConf();
+ conf.setVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES,
+ "overlay,reoptimize,reexecute_lost_am,dagsubmit,test");
+
+ DriverFactory.newDriver(conf);
+ }
+
+ @Test
+ public void testNormalAndCustom() {
+ HiveConf conf = new HiveConf();
+ conf.setVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES,
+ "overlay,reoptimize,reexecute_lost_am,dagsubmit" +
+ ",org.apache.hadoop.hive.ql.reexec.ReCompileWithoutCBOPlugin" +
+ ",org.apache.hadoop.hive.ql.reexec.ReExecuteOnWriteConflictPlugin");
+
+ IDriver driver = DriverFactory.newDriver(conf);
+ ReExecDriver reDriver = (ReExecDriver) driver;
+
+ List<IReExecutionPlugin> plugins = getPlugins(conf);
+ for (IReExecutionPlugin original : plugins) {
+ boolean found = false;
+ for (IReExecutionPlugin instance : reDriver.getPlugins()) {
+ if
(original.getClass().getName().equals(instance.getClass().getName())) {
+ found = true;
+ }
+ }
+
+ if (!found) {
+ fail("The ReExecutionPlugin defined has not been instantiated");
+ }
+ }
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testCustomNotInstanceOfIReExecutionPlugin() {
+ HiveConf conf = new HiveConf();
+ conf.setVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES,
+ "overlay,reoptimize,reexecute_lost_am,dagsubmit" +
+ ",org.apache.hadoop.hive.conf.HiveConf");
+
+ DriverFactory.newDriver(conf);
+ }
+
+ private List<IReExecutionPlugin> getPlugins(HiveConf conf) {
+ String strategies =
conf.getVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES);
+ List<IReExecutionPlugin> plugins = new ArrayList<>();
+ for (String string : strategies.split(",")) {
+ if (string.trim().isEmpty()) {
+ continue;
+ }
+
+ plugins.add(buildReExecPlugin(string));
+ }
+
+ return plugins;
+ }
+
+ private static IReExecutionPlugin buildReExecPlugin(String name) throws
RuntimeException {
+ Class<? extends IReExecutionPlugin> pluginType;
+ try {
+ pluginType = ReExecutionStrategyType.getPluginClassByName(name);
+ } catch (IllegalArgumentException e) {
+ try {
+ Class<?> cls = Class.forName(name);
+ if (cls.isAssignableFrom(IReExecutionPlugin.class)) {
+ throw new RuntimeException("Not re-execution plugin: " + name);
+ }
+
+ pluginType = (Class<? extends IReExecutionPlugin>) cls;
+ } catch (ClassNotFoundException e1) {
+ throw new RuntimeException(
+ "Unknown re-execution plugin: " + name + " (" +
ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES.varname + ")");
+ }
+ }
+
+ try {
+ return pluginType.getDeclaredConstructor(null).newInstance(null);
+ } catch (InstantiationException | IllegalAccessException |
InvocationTargetException | NoSuchMethodException e) {
+ throw new RuntimeException(
+ "Unknown re-execution plugin: " + name + " (" +
ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES.varname + ")");
+ }
+ }
+}