Repository: hive Updated Branches: refs/heads/master c9a5286e7 -> 8de3334c2
HIVE-18383: Qtests: running all cases from TestNegativeCliDriver results in OOMs (Zoltan Haindrich reviewed by Ashutosh Chauhan) Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8de3334c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8de3334c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8de3334c Branch: refs/heads/master Commit: 8de3334c2ace3da97bca8dce0536e28f36367d84 Parents: c9a5286 Author: Zoltan Haindrich <k...@rxd.hu> Authored: Fri Jan 26 13:17:00 2018 +0100 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Fri Jan 26 13:17:00 2018 +0100 ---------------------------------------------------------------------- .../hive/cli/control/AbstractCliConfig.java | 3 +- .../org/apache/hadoop/hive/ql/QTestUtil.java | 11 ++++-- .../hadoop/hive/ql/session/SessionState.java | 38 +++++++++++++------- 3 files changed, 36 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8de3334c/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java index 64db08a..cbba779 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java @@ -30,6 +30,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Properties; import java.util.Set; +import java.util.TreeSet; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; @@ -237,7 +238,7 @@ public abstract class AbstractCliConfig { File queryDir = new File(queryDirectory); // dedup file list - Set<File> testFiles = new LinkedHashSet<>(); + Set<File> testFiles = new TreeSet<>(); if (queryFile != null && !queryFile.equals("")) { // The user may have passed a list of files - comma separated for (String qFile : TEST_SPLITTER.split(queryFile)) { http://git-wip-us.apache.org/repos/asf/hive/blob/8de3334c/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 3f377f9..0b0b7ee 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -198,7 +198,6 @@ public class QTestUtil { private MiniLlapCluster llapCluster = null; private String hadoopVer = null; private QTestSetup setup = null; - private TezSessionState tezSessionState = null; private SparkSession sparkSession = null; private boolean isSessionStateStarted = false; private static final String javaVersion = getJavaVersion(); @@ -1192,7 +1191,7 @@ public class QTestUtil { boolean canReuseSession = !qNoSessionReuseQuerySet.contains(tname); if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) { // Copy the tezSessionState from the old CliSessionState. - tezSessionState = oldSs.getTezSession(); + TezSessionState tezSessionState = oldSs.getTezSession(); oldSs.setTezSession(null); ss.setTezSession(tezSessionState); oldSs.close(); @@ -1208,6 +1207,9 @@ public class QTestUtil { if (oldSs != null && oldSs.out != null && oldSs.out != System.out) { oldSs.out.close(); } + if (oldSs != null) { + oldSs.close(); + } SessionState.start(ss); cliDriver = new CliDriver(); @@ -1237,7 +1239,7 @@ public class QTestUtil { SessionState oldSs = SessionState.get(); if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) { // Copy the tezSessionState from the old CliSessionState. - tezSessionState = oldSs.getTezSession(); + TezSessionState tezSessionState = oldSs.getTezSession(); ss.setTezSession(tezSessionState); oldSs.setTezSession(null); oldSs.close(); @@ -1252,6 +1254,9 @@ public class QTestUtil { if (oldSs != null && oldSs.out != null && oldSs.out != System.out) { oldSs.out.close(); } + if (oldSs != null) { + oldSs.close(); + } SessionState.start(ss); isSessionStateStarted = true; http://git-wip-us.apache.org/repos/asf/hive/blob/8de3334c/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 8af19d8..d6083e6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -561,7 +561,9 @@ public class SessionState { public static void endStart(SessionState startSs) throws CancellationException, InterruptedException { - if (startSs.tezSessionState == null) return; + if (startSs.tezSessionState == null) { + return; + } startSs.tezSessionState.endOpen(); } @@ -617,7 +619,9 @@ public class SessionState { } String engine = HiveConf.getVar(startSs.getConf(), HiveConf.ConfVars.HIVE_EXECUTION_ENGINE); - if (!engine.equals("tez") || startSs.isHiveServerQuery) return; + if (!engine.equals("tez") || startSs.isHiveServerQuery) { + return; + } try { if (startSs.tezSessionState == null) { @@ -857,7 +861,7 @@ public class SessionState { fs.cancelDeleteOnExit(path); fs.delete(path, true); LOG.info("Deleted directory: {} on fs with scheme {}", path, fs.getScheme()); - } catch (IOException e) { + } catch (IllegalArgumentException | UnsupportedOperationException | IOException e) { LOG.error("Failed to delete path at {} on fs with scheme {}", path, (fs == null ? "Unknown-null" : fs.getScheme()), e); } @@ -1275,7 +1279,9 @@ public class SessionState { */ public void loadAuxJars() throws IOException { String[] jarPaths = StringUtils.split(sessionConf.getAuxJars(), ','); - if (ArrayUtils.isEmpty(jarPaths)) return; + if (ArrayUtils.isEmpty(jarPaths)) { + return; + } URLClassLoader currentCLoader = (URLClassLoader) SessionState.get().getConf().getClassLoader(); @@ -1708,7 +1714,9 @@ public class SessionState { } registry.clear(); - if (txnMgr != null) txnMgr.closeTxnManager(); + if (txnMgr != null) { + txnMgr.closeTxnManager(); + } JavaUtils.closeClassLoadersTo(sessionConf.getClassLoader(), parentLoader); File resourceDir = new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR)); @@ -1750,12 +1758,16 @@ public class SessionState { boolean isLocalMetastore = HiveConfUtil.isEmbeddedMetaStore(sessionConf.getVar(HiveConf.ConfVars.METASTOREURIS)); if (isLocalMetastore) { - if (sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL) - .equals(ObjectStore.class.getName()) || - sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL) - .equals(CachedStore.class.getName()) && sessionConf - .getVar(ConfVars.METASTORE_CACHED_RAW_STORE_IMPL) - .equals(ObjectStore.class.getName())) { + + String rawStoreImpl = sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL); + String realStoreImpl; + if (rawStoreImpl.equals(CachedStore.class.getName())) { + realStoreImpl = sessionConf.getVar(ConfVars.METASTORE_CACHED_RAW_STORE_IMPL); + } else { + realStoreImpl = rawStoreImpl; + } + Class<?> clazz = Class.forName(realStoreImpl); + if (ObjectStore.class.isAssignableFrom(clazz)) { ObjectStore.unCacheDataNucleusClassLoaders(); } } @@ -1823,7 +1835,9 @@ public class SessionState { /** Called from TezTask to attach a TezSession to use to the threadlocal. Ugly pattern... */ public void setTezSession(TezSessionState session) { - if (tezSessionState == session) return; // The same object. + if (tezSessionState == session) { + return; // The same object. + } if (tezSessionState != null) { tezSessionState.markFree(); tezSessionState.setKillQuery(null);