Repository: hive Updated Branches: refs/heads/llap 6e60a640c -> d6739a6b6
HIVE-11803: LLAP: MiniLlapCluster tests are slow Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d6739a6b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d6739a6b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d6739a6b Branch: refs/heads/llap Commit: d6739a6b64cd7e4bae48110cdba822424caa5952 Parents: 6e60a64 Author: Prasanth Jayachandran <j.prasant...@gmail.com> Authored: Fri Sep 11 16:19:40 2015 -0500 Committer: Prasanth Jayachandran <j.prasant...@gmail.com> Committed: Fri Sep 11 16:19:40 2015 -0500 ---------------------------------------------------------------------- data/conf/llap/hive-site.xml | 10 ---------- data/conf/tez/hive-site.xml | 5 ----- .../hadoop/hive/ql/TestLocationQueries.java | 2 +- .../org/apache/hadoop/hive/ql/QTestUtil.java | 21 +++++++------------- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 10 +++++----- 5 files changed, 13 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/d6739a6b/data/conf/llap/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml index c002b1e..0e8c7c1 100644 --- a/data/conf/llap/hive-site.xml +++ b/data/conf/llap/hive-site.xml @@ -39,16 +39,6 @@ </property> <property> - <name>hive.execution.mode</name> - <value>container</value> -</property> - -<property> - <name>mapred.tez.java.opts</name> - <value>-Xmx128m</value> -</property> - -<property> <name>hive.tez.container.size</name> <value>128</value> <description></description> http://git-wip-us.apache.org/repos/asf/hive/blob/d6739a6b/data/conf/tez/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/tez/hive-site.xml b/data/conf/tez/hive-site.xml index b4abe90..8acf270 100644 --- a/data/conf/tez/hive-site.xml +++ b/data/conf/tez/hive-site.xml @@ -39,11 +39,6 @@ </property> <property> - <name>mapred.tez.java.opts</name> - <value>-Xmx128m</value> -</property> - -<property> <name>hive.tez.container.size</name> <value>128</value> <description></description> http://git-wip-us.apache.org/repos/asf/hive/blob/d6739a6b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java index 1fe4f48..6ee98cb 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java @@ -88,7 +88,7 @@ public class TestLocationQueries extends BaseTestQueries { String hadoopVer, String locationSubdir) throws Exception { - super(outDir, logDir, miniMr, null, hadoopVer, "", "", false); + super(outDir, logDir, miniMr, null, hadoopVer, "", ""); this.locationSubdir = locationSubdir; } } http://git-wip-us.apache.org/repos/asf/hive/blob/d6739a6b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 68141ea..0d74d7f 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -347,15 +347,8 @@ public class QTestUtil { } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, - String confDir, String hadoopVer, String initScript, - String cleanupScript) throws Exception { - this(outDir, logDir, clusterType, confDir, - hadoopVer, initScript, cleanupScript, true); - } - - public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, - String cleanupScript, boolean withLlapIo) throws Exception { + String cleanupScript) throws Exception { this.outDir = outDir; this.logDir = logDir; @@ -446,7 +439,7 @@ public class QTestUtil { cleanUp(); } - if (clusterType == MiniClusterType.tez) { + if (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap) { SessionState.get().getTezSession().close(false); } setup.tearDown(); @@ -912,8 +905,8 @@ public class QTestUtil { ss.setIsSilent(true); SessionState oldSs = SessionState.get(); - if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.spark - || clusterType == MiniClusterType.miniSparkOnYarn)) { + if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap + || clusterType == MiniClusterType.spark || clusterType == MiniClusterType.miniSparkOnYarn)) { sparkSession = oldSs.getSparkSession(); ss.setSparkSession(sparkSession); oldSs.setSparkSession(null); @@ -976,8 +969,8 @@ public class QTestUtil { ss.err = System.out; SessionState oldSs = SessionState.get(); - if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.spark - || clusterType == MiniClusterType.miniSparkOnYarn)) { + if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap + || clusterType == MiniClusterType.miniSparkOnYarn || clusterType == MiniClusterType.miniSparkOnYarn)) { sparkSession = oldSs.getSparkSession(); ss.setSparkSession(sparkSession); oldSs.setSparkSession(null); @@ -1838,7 +1831,7 @@ public class QTestUtil { QTestUtil[] qt = new QTestUtil[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", - defaultInitScript, defaultCleanupScript, false); + defaultInitScript, defaultCleanupScript); qt[i].addFile(qfiles[i]); qt[i].clearTestSideEffects(); } http://git-wip-us.apache.org/repos/asf/hive/blob/d6739a6b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java ---------------------------------------------------------------------- diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index e650cfd..876754d 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -390,11 +390,6 @@ public class Hadoop23Shims extends HadoopShimsSecure { public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, boolean isLlap) throws IOException { - if (isLlap) { - createAndLaunchLlapDaemon(conf); - } else { - miniLlapCluster = null; - } mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); @@ -402,6 +397,11 @@ public class Hadoop23Shims extends HadoopShimsSecure { mr.init(conf); mr.start(); this.conf = mr.getConfig(); + if (isLlap) { + createAndLaunchLlapDaemon(this.conf); + } else { + miniLlapCluster = null; + } } private void createAndLaunchLlapDaemon(final Configuration conf) {