HIVE-14824. Separate fstype from cluster type in QTestUtil. (Siddharth Seth, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cf72a737 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cf72a737 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cf72a737 Branch: refs/heads/repl2 Commit: cf72a73708b00b2e585d101258d95eb9cbd3791f Parents: e2bd513 Author: Siddharth Seth <ss...@apache.org> Authored: Wed Sep 28 13:40:04 2016 -0700 Committer: Siddharth Seth <ss...@apache.org> Committed: Wed Sep 28 13:40:04 2016 -0700 ---------------------------------------------------------------------- .../hive/cli/control/AbstractCliConfig.java | 10 +++ .../hadoop/hive/cli/control/CliConfigs.java | 12 ++- .../hadoop/hive/cli/control/CoreCliDriver.java | 3 +- .../org/apache/hadoop/hive/ql/QTestUtil.java | 84 +++++++++++--------- 4 files changed, 69 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/cf72a737/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java index efbd465..03d4075 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java @@ -35,6 +35,7 @@ import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.QTestUtil; +import org.apache.hadoop.hive.ql.QTestUtil.FsType; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import com.google.common.base.Splitter; import com.google.common.collect.Sets; @@ -63,6 +64,7 @@ public abstract class AbstractCliConfig { private String initScript; private String hiveConfDir; private MiniClusterType clusterType; + private FsType fsType; // FIXME: null value is treated differently on the other end..when those filter will be // moved...this may change @@ -380,6 +382,14 @@ public abstract class AbstractCliConfig { } } + protected FsType getFsType() { + return this.fsType; + } + + protected void setFsType(FsType fsType) { + this.fsType = fsType; + } + private String getSysPropValue(String propName) { String propValue = System.getProperty(propName); if (propValue == null || propValue.trim().length() == 0) { http://git-wip-us.apache.org/repos/asf/hive/blob/cf72a737/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index 0068b95..ca72282 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -21,6 +21,7 @@ import java.io.File; import java.net.MalformedURLException; import java.net.URL; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.ql.parse.CoreParseNegative; @@ -171,8 +172,15 @@ public class CliConfigs { setInitScript("q_test_init_for_encryption.sql"); setCleanupScript("q_test_cleanup_for_encryption.sql"); - setHiveConfDir("data/conf"); - setClusterType(MiniClusterType.encrypted); + + setClusterType(MiniClusterType.mr); + setFsType(QTestUtil.FsType.encrypted_hdfs); + if (getClusterType() == MiniClusterType.tez) { + setHiveConfDir("data/conf/tez"); + } else { + setHiveConfDir("data/conf"); + } + } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); } http://git-wip-us.apache.org/repos/asf/hive/blob/cf72a737/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index db58f1d..d83ff45 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -61,7 +61,8 @@ public class CoreCliDriver extends CliAdapter { @Override public QTestUtil invokeInternal() throws Exception { return new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR, - hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true); + hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true, false, + cliConfig.getFsType()); } }.invoke("QtestUtil instance created", LOG, true); http://git-wip-us.apache.org/repos/asf/hive/blob/cf72a737/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 0dfd727..909d7f6 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -51,7 +51,6 @@ import java.util.Collection; import java.util.Comparator; import java.util.Deque; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -86,9 +85,7 @@ import org.apache.hadoop.hive.common.io.SortPrintStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.LlapItUtils; -import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; -import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Index; @@ -155,7 +152,6 @@ public class QTestUtil { private String testWarehouse; private final String testFiles; - private final boolean useLocalFs; private final boolean localMode; protected final String outDir; protected final String logDir; @@ -170,6 +166,7 @@ public class QTestUtil { private static final String SORT_SUFFIX = ".sorted"; private final HashSet<String> srcTables; private final MiniClusterType clusterType; + private final FsType fsType; private ParseDriver pd; protected Hive db; protected QueryState queryState; @@ -414,14 +411,32 @@ public class QTestUtil { } } + + public enum FsType { + local, + hdfs, + encrypted_hdfs, + } + public enum MiniClusterType { - mr, - tez, - spark, - encrypted, - miniSparkOnYarn, - llap, - none; + + mr(FsType.hdfs), + tez(FsType.hdfs), + spark(FsType.local), + miniSparkOnYarn(FsType.hdfs), + llap(FsType.hdfs), + none(FsType.local); + + + private final FsType defaultFsType; + + MiniClusterType(FsType defaultFsType) { + this.defaultFsType = defaultFsType; + } + + public FsType getDefaultFsType() { + return defaultFsType; + } public static MiniClusterType valueForString(String type) { if (type.equals("miniMR")) { @@ -430,8 +445,6 @@ public class QTestUtil { return tez; } else if (type.equals("spark")) { return spark; - } else if (type.equals("encrypted")) { - return encrypted; } else if (type.equals("miniSparkOnYarn")) { return miniSparkOnYarn; } else if (type.equals("llap")) { @@ -442,6 +455,7 @@ public class QTestUtil { } } + private String getKeyProviderURI() { // Use the target directory if it is not specified String HIVE_ROOT = AbstractCliConfig.HIVE_ROOT; @@ -473,19 +487,24 @@ public class QTestUtil { String confDir, String hadoopVer, String initScript, String cleanupScript, boolean useHBaseMetastore, boolean withLlapIo) throws Exception { this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, - useHBaseMetastore, withLlapIo, false, false); + useHBaseMetastore, withLlapIo, false, null); } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript, - boolean useHBaseMetastore, boolean withLlapIo, boolean localMode, boolean useLocalFs) + boolean useHBaseMetastore, boolean withLlapIo, boolean localMode, FsType fsType) throws Exception { LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," + " hadoopVer={}, initScript={}, cleanupScript={}, useHbaseMetaStore={}, withLlapIo={}," + - " localMode={}, useLocalFs={}" + " localMode={}, fsType={}" , outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, - useHBaseMetastore, withLlapIo, localMode, useLocalFs); - this.useLocalFs = useLocalFs; + useHBaseMetastore, withLlapIo, localMode, fsType); + Preconditions.checkNotNull(clusterType, "ClusterType cannot be null"); + if (fsType != null) { + this.fsType = fsType; + } else { + this.fsType = clusterType.getDefaultFsType(); + } this.localMode = localMode; this.outDir = outDir; this.logDir = logDir; @@ -555,16 +574,12 @@ public class QTestUtil { private void setupFileSystem(HadoopShims shims) throws IOException { - if (useLocalFs) { - Preconditions - .checkState(clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap, - "useLocalFs can currently only be set for tez or llap"); - } - - if (clusterType != MiniClusterType.none && clusterType != MiniClusterType.spark) { + if (fsType == FsType.local) { + fs = FileSystem.getLocal(conf); + } else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs) { int numDataNodes = 4; - if (clusterType == MiniClusterType.encrypted) { + if (fsType == FsType.encrypted_hdfs) { // Set the security key provider so that the MiniDFS cluster is initialized // with encryption conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); @@ -578,16 +593,11 @@ public class QTestUtil { LOG.info("key provider is initialized"); } else { - if (!useLocalFs) { - dfs = shims.getMiniDfs(conf, numDataNodes, true, null); - fs = dfs.getFileSystem(); - } else { - fs = FileSystem.getLocal(conf); - } + dfs = shims.getMiniDfs(conf, numDataNodes, true, null); + fs = dfs.getFileSystem(); } } else { - // Setup local file system - fs = FileSystem.getLocal(conf); + throw new IllegalArgumentException("Unknown or unhandled fsType [" + fsType + "]"); } } @@ -621,7 +631,7 @@ public class QTestUtil { } } else if (clusterType == MiniClusterType.miniSparkOnYarn) { mr = shims.getMiniSparkCluster(conf, 4, uriString, 1); - } else if (clusterType == MiniClusterType.mr || clusterType == MiniClusterType.encrypted) { + } else if (clusterType == MiniClusterType.mr) { mr = shims.getMiniMrCluster(conf, 4, uriString, 1); } } @@ -900,7 +910,7 @@ public class QTestUtil { if(tblObj.isIndexTable()) { continue; } - db.dropTable(dbName, tblName, true, true, clusterType == MiniClusterType.encrypted); + db.dropTable(dbName, tblName, true, true, fsType == FsType.encrypted_hdfs); } else { // this table is defined in srcTables, drop all indexes on it List<Index> indexes = db.getIndexes(dbName, tblName, (short)-1); @@ -1563,7 +1573,7 @@ public class QTestUtil { boolean partialMaskWasMatched = false; Matcher matcher; while (null != (line = in.readLine())) { - if (clusterType == MiniClusterType.encrypted) { + if (fsType == FsType.encrypted_hdfs) { for (Pattern pattern : partialReservedPlanMask) { matcher = pattern.matcher(line); if (matcher.find()) {