hive git commit: HIVE-18038: org.apache.hadoop.hive.ql.session.OperationLog - Review (BELUGA BEHR, reviewed by Peter Vary)
Repository: hive Updated Branches: refs/heads/master c8b3f9aa0 -> c399c4ed1 HIVE-18038: org.apache.hadoop.hive.ql.session.OperationLog - Review (BELUGA BEHR, reviewed by Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c399c4ed Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c399c4ed Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c399c4ed Branch: refs/heads/master Commit: c399c4ed108c631b34570e1253e37e3075b0e7e3 Parents: c8b3f9a Author: Aihua Xu Authored: Tue Sep 11 10:27:46 2018 -0700 Committer: Aihua Xu Committed: Tue Sep 11 10:27:46 2018 -0700 -- .../hadoop/hive/ql/session/OperationLog.java| 117 ++- 1 file changed, 65 insertions(+), 52 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/c399c4ed/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java index 6d75c29..c48dc42 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java @@ -17,25 +17,35 @@ */ package org.apache.hadoop.hive.ql.session; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - /** * OperationLog wraps the actual operation log file, and provides interface * for accessing, reading, writing, and removing the file. */ public class OperationLog { - private static final Logger LOG = LoggerFactory.getLogger(OperationLog.class.getName()); + private static final Logger LOG = LoggerFactory.getLogger(OperationLog.class); private final String operationName; + private final LogFile logFile; // If in test mode then the LogDivertAppenderForTest created an extra log file containing only // the output needed for the qfile results. @@ -45,7 +55,8 @@ public class OperationLog { private final boolean isShortLogs; // True if the logs should be removed after the operation. Should be used only in test mode private final boolean isRemoveLogs; - private LoggingLevel opLoggingLevel = LoggingLevel.UNKNOWN; + + private final LoggingLevel opLoggingLevel; public enum LoggingLevel { NONE, EXECUTION, PERFORMANCE, VERBOSE, UNKNOWN @@ -58,6 +69,8 @@ public class OperationLog { if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED)) { String logLevel = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL); opLoggingLevel = getLoggingLevel(logLevel); +} else { + opLoggingLevel = LoggingLevel.UNKNOWN; } // If in test mod create a test log file which will contain only logs which are supposed to @@ -79,15 +92,17 @@ public class OperationLog { } public static LoggingLevel getLoggingLevel (String mode) { -if (mode.equalsIgnoreCase("none")) { +String m = StringUtils.defaultString(mode).toLowerCase(); +switch (m) { +case "none": return LoggingLevel.NONE; -} else if (mode.equalsIgnoreCase("execution")) { +case "execution": return LoggingLevel.EXECUTION; -} else if (mode.equalsIgnoreCase("verbose")) { +case "verbose": return LoggingLevel.VERBOSE; -} else if (mode.equalsIgnoreCase("performance")) { +case "performance": return LoggingLevel.PERFORMANCE; -} else { +default: return LoggingLevel.UNKNOWN; } } @@ -105,11 +120,8 @@ public class OperationLog { */ public List readOperationLog(boolean isFetchFirst, long maxRows) throws SQLException { -if (isShortLogs) { - return testLogFile.read(isFetchFirst, maxRows); -} else { - return logFile.read(isFetchFirst, maxRows); -} +LogFile lf = (isShortLogs) ? testLogFile : logFile; +return lf.read(isFetchFirst, maxRows); } /** @@ -144,8 +156,10 @@ public class OperationLog { if (isFetchFirst) { resetIn(); } - - return readResults(maxRows); + if (maxRows >= (long) Int
hive git commit: HIVE-20020: Hive contrib jar should not be in lib (Alice Fan, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master b60b13f97 -> afb61aebf HIVE-20020: Hive contrib jar should not be in lib (Alice Fan, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/afb61aeb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/afb61aeb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/afb61aeb Branch: refs/heads/master Commit: afb61aebfd475e2cb4f7935f94f765ae53bb174f Parents: b60b13f Author: Aihua Xu Authored: Mon Sep 10 11:31:16 2018 -0700 Committer: Aihua Xu Committed: Mon Sep 10 11:35:14 2018 -0700 -- packaging/src/main/assembly/bin.xml | 10 ++ 1 file changed, 10 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/afb61aeb/packaging/src/main/assembly/bin.xml -- diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml index eeed3ec..fceb1be 100644 --- a/packaging/src/main/assembly/bin.xml +++ b/packaging/src/main/assembly/bin.xml @@ -47,6 +47,7 @@ co.cask.tephra:* commons-configuration:commons-configuration org.apache.hive:hive-jdbc:*:standalone +org.apache.hive:hive-contrib @@ -59,6 +60,15 @@ + contrib/ + false + false + true + +org.apache.hive:hive-contrib + + + lib false false
[1/2] hive git commit: HIVE-20237: Do Not Print StackTraces to STDERR in HiveMetaStore (Alice Fan, reviewed Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 4408661c0 -> ff2554328 HIVE-20237: Do Not Print StackTraces to STDERR in HiveMetaStore (Alice Fan, reviewed Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1f9c70e5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1f9c70e5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1f9c70e5 Branch: refs/heads/master Commit: 1f9c70e5148add278a1cb9b2186fb9a65d7ed1c5 Parents: 4408661 Author: Aihua Xu Authored: Wed Aug 22 10:02:08 2018 -0700 Committer: Aihua Xu Committed: Wed Aug 22 10:02:08 2018 -0700 -- .../hadoop/hive/metastore/HiveMetaStore.java| 280 +-- 1 file changed, 137 insertions(+), 143 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/1f9c70e5/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java -- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 324035a..067eb5a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -9219,166 +9219,160 @@ public class HiveMetaStore extends ThriftHiveMetastore { public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, Configuration conf, Lock startLock, Condition startCondition, AtomicBoolean startedServing) throws Throwable { -try { - isMetaStoreRemote = true; - // Server will create new threads up to max as necessary. After an idle - // period, it will destroy threads to keep the number of threads in the - // pool to min. - long maxMessageSize = MetastoreConf.getLongVar(conf, ConfVars.SERVER_MAX_MESSAGE_SIZE); - int minWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MIN_THREADS); - int maxWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MAX_THREADS); - boolean tcpKeepAlive = MetastoreConf.getBoolVar(conf, ConfVars.TCP_KEEP_ALIVE); - boolean useFramedTransport = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_FRAMED_TRANSPORT); - boolean useCompactProtocol = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_COMPACT_PROTOCOL); - boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL); - useSasl = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_SASL); - - if (useSasl) { -// we are in secure mode. Login using keytab -String kerberosName = SecurityUtil -.getServerPrincipal(MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL), "0.0.0.0"); -String keyTabFile = MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE); -UserGroupInformation.loginUserFromKeytab(kerberosName, keyTabFile); - } - - TProcessor processor; - TTransportFactory transFactory; - final TProtocolFactory protocolFactory; - final TProtocolFactory inputProtoFactory; - if (useCompactProtocol) { -protocolFactory = new TCompactProtocol.Factory(); -inputProtoFactory = new TCompactProtocol.Factory(maxMessageSize, maxMessageSize); +isMetaStoreRemote = true; +// Server will create new threads up to max as necessary. After an idle +// period, it will destroy threads to keep the number of threads in the +// pool to min. +long maxMessageSize = MetastoreConf.getLongVar(conf, ConfVars.SERVER_MAX_MESSAGE_SIZE); +int minWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MIN_THREADS); +int maxWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MAX_THREADS); +boolean tcpKeepAlive = MetastoreConf.getBoolVar(conf, ConfVars.TCP_KEEP_ALIVE); +boolean useFramedTransport = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_FRAMED_TRANSPORT); +boolean useCompactProtocol = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_COMPACT_PROTOCOL); +boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL); +useSasl = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_SASL); + +if (useSasl) { + // we are in secure mode. Login using keytab + String kerberosName = SecurityUtil + .getServerPrincipal(MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL), "0.0.0.0"); + String keyTabFile = MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE); + UserGroupInformation.loginUserFromKeytab(kerberosName, keyTabFile); +} + +TProcessor processor; +TTransportFactory transFactory; +f
[2/2] hive git commit: HIVE-20246: Configurable collecting stats by using DO_NOT_UPDATE_STATS table property (Alice Fan, reviewed by Aihua Xu)
HIVE-20246: Configurable collecting stats by using DO_NOT_UPDATE_STATS table property (Alice Fan, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ff255432 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ff255432 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ff255432 Branch: refs/heads/master Commit: ff25543282532613c9cfa768a8bf8ee728b3734e Parents: 1f9c70e Author: Aihua Xu Authored: Wed Aug 22 10:05:12 2018 -0700 Committer: Aihua Xu Committed: Wed Aug 22 10:05:12 2018 -0700 -- .../hadoop/hive/metastore/HiveMetaStore.java| 27 -- .../hive/metastore/TestHiveMetaStore.java | 57 +++- 2 files changed, 79 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/ff255432/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java -- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 067eb5a..e971d0f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -3174,8 +3174,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { part.setCreateTime((int) time); part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time)); -if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) && -!MetaStoreServerUtils.isView(tbl)) { +if (canUpdateStats(tbl)) { MetaStoreServerUtils.updatePartitionStatsFast(part, tbl, wh, madeDir, false, envContext, true); } @@ -3791,6 +3790,27 @@ public class HiveMetaStore extends ThriftHiveMetastore { return result; } +/** + * Verify if update stats while altering partition(s) + * For the following three cases HMS will not update partition stats + * 1) Table property 'DO_NOT_UPDATE_STATS' = True + * 2) HMS configuration property 'STATS_AUTO_GATHER' = False + * 3) Is View + */ +private boolean canUpdateStats(Table tbl) { +Map tblParams = tbl.getParameters(); +boolean updateStatsTbl = true; +if ((tblParams != null) && tblParams.containsKey(StatsSetupConst.DO_NOT_UPDATE_STATS)) { +updateStatsTbl = !Boolean.valueOf(tblParams.get(StatsSetupConst.DO_NOT_UPDATE_STATS)); +} +if (!MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) || +MetaStoreServerUtils.isView(tbl) || +!updateStatsTbl) { + return false; +} +return true; +} + private void initializeAddedPartition( final Table tbl, final Partition part, boolean madeDir) throws MetaException { initializeAddedPartition(tbl, new PartitionSpecProxy.SimplePartitionWrapperIterator(part), madeDir); @@ -3798,8 +3818,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { private void initializeAddedPartition( final Table tbl, final PartitionSpecProxy.PartitionIterator part, boolean madeDir) throws MetaException { - if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) && - !MetaStoreServerUtils.isView(tbl)) { + if (canUpdateStats(tbl)) { MetaStoreServerUtils.updatePartitionStatsFast(part, tbl, wh, madeDir, false, null, true); } http://git-wip-us.apache.org/repos/asf/hive/blob/ff255432/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java -- diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 60beab6..4937d9d 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.metastore; -import java.lang.reflect.Field; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; @@ -38,6 +37,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.lang.reflect.*; +import static org.m
hive git commit: HIVE-20331: Query with union all, lateral view and Join fails with "cannot find parent in the child operator" (Aihua Xu, reviewed by Vihang Karajgaonkar)
Repository: hive Updated Branches: refs/heads/master ddc246528 -> 61372dfaf HIVE-20331: Query with union all, lateral view and Join fails with "cannot find parent in the child operator" (Aihua Xu, reviewed by Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/61372dfa Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/61372dfa Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/61372dfa Branch: refs/heads/master Commit: 61372dfafa35b43be710554e4920e3b6ed4dacdf Parents: ddc2465 Author: Aihua Xu Authored: Wed Aug 8 09:07:31 2018 -0700 Committer: Aihua Xu Committed: Thu Aug 16 10:57:21 2018 -0700 -- .../hive/ql/optimizer/GenMRProcContext.java | 11 ++ .../hive/ql/optimizer/GenMRTableScan1.java | 2 + .../clientpositive/unionall_lateralview.q | 29 + .../clientpositive/unionall_lateralview.q.out | 105 +++ 4 files changed, 147 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/61372dfa/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java index f80395d..782ce16 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java @@ -219,6 +219,17 @@ public class GenMRProcContext implements NodeProcessorCtx { } /** + * The context is reused across the rules. Reset so the following info is not + * incorrectly carried over to the following optimizations starting with the new TS. + */ + public void reset() { +currTask = null; +currTopOp = null; +currUnionOp = null; +currAliasId = null; + } + + /** * @return reducer to task mapping */ public HashMap, http://git-wip-us.apache.org/repos/asf/hive/blob/61372dfa/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java index 6295d7f..bb53ce8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java @@ -66,6 +66,8 @@ public class GenMRTableScan1 implements NodeProcessor { Object... nodeOutputs) throws SemanticException { TableScanOperator op = (TableScanOperator) nd; GenMRProcContext ctx = (GenMRProcContext) opProcCtx; +ctx.reset(); + ParseContext parseCtx = ctx.getParseCtx(); Table table = op.getConf().getTableMetadata(); Class inputFormat = table.getInputFormatClass(); http://git-wip-us.apache.org/repos/asf/hive/blob/61372dfa/ql/src/test/queries/clientpositive/unionall_lateralview.q -- diff --git a/ql/src/test/queries/clientpositive/unionall_lateralview.q b/ql/src/test/queries/clientpositive/unionall_lateralview.q new file mode 100644 index 000..457f404 --- /dev/null +++ b/ql/src/test/queries/clientpositive/unionall_lateralview.q @@ -0,0 +1,29 @@ +-- Test the case HIVE-20331 with the union all, lateral view and join +DROP TABLE IF EXISTS unionall_lateralview1; +DROP TABLE IF EXISTS unionall_lateralview2; +CREATE TABLE unionall_lateralview1(col1 INT); +INSERT INTO unionall_lateralview1 VALUES(1), (2); +CREATE TABLE unionall_lateralview2(col1 INT); + +INSERT INTO unionall_lateralview2 +SELECT 1 AS `col1` +FROM unionall_lateralview1 +UNION ALL + SELECT 2 AS `col1` + FROM +(SELECT col1 + FROM unionall_lateralview1 +) x1 +JOIN + (SELECT col1 + FROM +(SELECT + Row_Number() over (PARTITION BY col1 ORDER BY col1) AS `col1` +FROM unionall_lateralview1 +) x2 lateral VIEW explode(map(10,1))`mapObj` AS `col2`, `col3` + ) `expdObj`; + +SELECT * FROM unionall_lateralview2 ORDER BY col1; + +DROP TABLE unionall_lateralview1; +DROP TABLE unionall_lateralview2; http://git-wip-us.apache.org/repos/asf/hive/blob/61372dfa/ql/src/test/results/clientpositive/unionall_lateralview.q.out -- diff --git a/ql/src/test/results/clientpositive/unionall_lateralview.q.out b/ql/src/test/results/clientpositive/unionall_lateralview.q.out new file mode 100644 index 000..db64777 --- /dev/null +++ b/ql/src/test/results/clientpositive/unionall_lateralview.q.out @@ -0,0 +1,105 @@ +PREHOOK: query: DROP TABLE IF EXISTS unionall_lateralview1 +PREHOOK: type: DROPTABLE +POST
hive git commit: HIVE-20345: Drop database may hang if the tables get deleted from a different call (Aihua Xu, reviewed by Naveen Gangam)
Repository: hive Updated Branches: refs/heads/master 5f993e24c -> 86096b48f HIVE-20345: Drop database may hang if the tables get deleted from a different call (Aihua Xu, reviewed by Naveen Gangam) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/86096b48 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/86096b48 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/86096b48 Branch: refs/heads/master Commit: 86096b48f8ead2f228725c49953c60b2f852174b Parents: 5f993e2 Author: Aihua Xu Authored: Wed Aug 8 18:37:54 2018 -0700 Committer: Aihua Xu Committed: Tue Aug 14 14:48:58 2018 -0700 -- .../java/org/apache/hadoop/hive/metastore/HiveMetaStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/86096b48/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java -- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 95b08eb..324035a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1573,9 +1573,9 @@ public class HiveMetaStore extends ThriftHiveMetastore { drop_table(MetaStoreUtils.prependCatalogToDbName(table.getCatName(), table.getDbName(), conf), table.getTableName(), false); } - -startIndex = endIndex; } + + startIndex = endIndex; } if (ms.dropDatabase(catName, name)) {
hive git commit: HIVE-20136: Code Review of ArchiveUtils Class (BELUGA BEHR, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 0fd23b6bd -> 7dce7b79b HIVE-20136: Code Review of ArchiveUtils Class (BELUGA BEHR, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7dce7b79 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7dce7b79 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7dce7b79 Branch: refs/heads/master Commit: 7dce7b79b9d877256046c5e595110a23de9bdcce Parents: 0fd23b6 Author: Aihua Xu Authored: Wed Aug 8 10:40:14 2018 -0700 Committer: Aihua Xu Committed: Wed Aug 8 10:40:14 2018 -0700 -- .../hadoop/hive/ql/exec/ArchiveUtils.java | 78 1 file changed, 32 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/7dce7b79/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java index 5576d11..6ad0556 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java @@ -21,13 +21,14 @@ package org.apache.hadoop.hive.ql.exec; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.Warehouse; @@ -41,7 +42,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; /** * ArchiveUtils. - * */ @SuppressWarnings("nls") public final class ArchiveUtils { @@ -105,7 +105,7 @@ public final class ArchiveUtils { throw new HiveException("Unable to get partitions directories prefix", e); } Path tableDir = tbl.getDataLocation(); - if(tableDir == null) { + if (tableDir == null) { throw new HiveException("Table has no location set"); } return new Path(tableDir, prefixSubdir); @@ -136,16 +136,16 @@ public final class ArchiveUtils { public HarPathHelper(HiveConf hconf, URI archive, URI originalBase) throws HiveException { this.originalBase = addSlash(originalBase); String parentHost = archive.getHost(); - String harHost = null; + String harHost = archive.getScheme(); if (parentHost == null) { -harHost = archive.getScheme() + "-localhost"; +harHost += "-localhost"; } else { -harHost = archive.getScheme() + "-" + parentHost; +harHost += "-" + parentHost; } // have to make sure there's slash after .har, otherwise resolve doesn't work - String path = addSlash(archive.getPath()); - if(!path.endsWith(".har/")) { + String path = StringUtils.appendIfMissing(archive.getPath(), "/"); + if (!path.endsWith(".har/")) { throw new HiveException("HAR archive path must end with .har"); } // harUri is used to access the partition's files, which are in the archive @@ -155,41 +155,33 @@ public final class ArchiveUtils { base = new URI("har", archive.getUserInfo(), harHost, archive.getPort(), path, archive.getQuery(), archive.getFragment()); } catch (URISyntaxException e) { -throw new HiveException("Couldn't create har URI from archive URI", e); +throw new HiveException("Could not create har URI from archive URI", e); } } public URI getHarUri(URI original) throws URISyntaxException { URI relative = originalBase.relativize(original); if (relative.isAbsolute()) { -throw new URISyntaxException("Couldn't create URI for location.", - "Relative: " + relative + " Base: " - + base + " OriginalBase: " + originalBase); +throw new URISyntaxException("Could not create URI for location.", +"Relative: " + relative + " Base: " + base + " OriginalBase: " ++ originalBase); } - return base.resolve(relative); - - } } - public static String addSlash(String s) { -return s.endsWith("/") ? s : s + "/"; - } - /** * Makes sure, that URI points to directory by adding slash to it. * Useful in relativizing URIs. */ public static URI addSlash(URI u) throws HiveException { -if(u.getPath().endsWith("/")) { +if (u.getPath().endsWith("/")) { return u; -} else { - try { -return new URI(u.getScheme(), u.getAuthority()
hive git commit: HIVE-19809: Remove Deprecated Code From Utilities Class (BELUGA BEHR, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 1c8449cce -> 109ec31dc HIVE-19809: Remove Deprecated Code From Utilities Class (BELUGA BEHR, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/109ec31d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/109ec31d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/109ec31d Branch: refs/heads/master Commit: 109ec31dc7de5678106e702c9bdd641c12cb1653 Parents: 1c8449c Author: Aihua Xu Authored: Fri Jul 13 15:47:34 2018 -0700 Committer: Aihua Xu Committed: Mon Jul 30 11:05:26 2018 -0700 -- .../org/apache/hadoop/hive/ql/QTestUtil.java| 4 ++- .../apache/hadoop/hive/ql/exec/Utilities.java | 26 2 files changed, 3 insertions(+), 27 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/109ec31d/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java -- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 4ed062e..5adbb63 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -36,6 +36,8 @@ import java.io.StringWriter; import java.io.UnsupportedEncodingException; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; @@ -1834,7 +1836,7 @@ public class QTestUtil { if (zooKeeperCluster == null) { //create temp dir String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY); -File tmpDir = Utilities.createTempDir(tmpBaseDir); +File tmpDir = Files.createTempDirectory(Paths.get(tmpBaseDir), "tmp_").toFile(); zooKeeperCluster = new MiniZooKeeperCluster(); zkPort = zooKeeperCluster.startup(tmpDir); http://git-wip-us.apache.org/repos/asf/hive/blob/109ec31d/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 80478ca..b677d46 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -3758,32 +3758,6 @@ public final class Utilities { } /** - * Create a temp dir in specified baseDir - * This can go away once hive moves to support only JDK 7 - * and can use Files.createTempDirectory - * Guava Files.createTempDir() does not take a base dir - * @param baseDir - directory under which new temp dir will be created - * @return File object for new temp dir - */ - public static File createTempDir(String baseDir){ -//try creating the temp dir MAX_ATTEMPTS times -final int MAX_ATTEMPS = 30; -for(int i = 0; i < MAX_ATTEMPS; i++){ - //pick a random file name - String tempDirName = "tmp_" + ((int)(10 * Math.random())); - - //return if dir could successfully be created with that file name - File tempDir = new File(baseDir, tempDirName); - if(tempDir.mkdir()){ -return tempDir; - } -} -throw new IllegalStateException("Failed to create a temp dir under " -+ baseDir + " Giving up after " + MAX_ATTEMPS + " attempts"); - - } - - /** * Skip header lines in the table file when reading the record. * * @param currRecReader
hive git commit: HIVE-20153: Count and Sum UDF consume more memory in Hive 2+ (Aihua Xu, reviewed by Gopal V)
Repository: hive Updated Branches: refs/heads/master 0ad71121d -> 804b125e7 HIVE-20153: Count and Sum UDF consume more memory in Hive 2+ (Aihua Xu, reviewed by Gopal V) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/804b125e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/804b125e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/804b125e Branch: refs/heads/master Commit: 804b125e743516ca0139a22c42244b980ffaf47f Parents: 0ad7112 Author: Aihua Xu Authored: Thu Jul 26 13:05:31 2018 -0700 Committer: Aihua Xu Committed: Fri Jul 27 13:41:16 2018 -0700 -- .../apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java | 6 +- .../apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java | 9 ++--- 2 files changed, 11 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/804b125e/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java index 2d7cc8d..c2414d2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java @@ -153,7 +153,7 @@ public class GenericUDAFCount implements GenericUDAFResolver2 { @Override public void reset(AggregationBuffer agg) throws HiveException { ((CountAgg) agg).value = 0; - ((CountAgg) agg).uniqueObjects = new HashSet(); + ((CountAgg) agg).uniqueObjects = null; } @Override @@ -177,7 +177,11 @@ public class GenericUDAFCount implements GenericUDAFResolver2 { // Skip the counting if the values are the same for windowing COUNT(DISTINCT) case if (countThisRow && isWindowingDistinct()) { + if (((CountAgg) agg).uniqueObjects == null) { +((CountAgg) agg).uniqueObjects = new HashSet(); + } HashSet uniqueObjs = ((CountAgg) agg).uniqueObjects; + ObjectInspectorObject obj = new ObjectInspectorObject( ObjectInspectorUtils.copyToStandardObject(parameters, inputOI, ObjectInspectorCopyOption.JAVA), outputOI); http://git-wip-us.apache.org/repos/asf/hive/blob/804b125e/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java index 1439b64..e30b903 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java @@ -184,6 +184,9 @@ public class GenericUDAFSum extends AbstractGenericUDAFResolver { } if (isWindowingDistinct()) { +if (agg.uniqueObjects == null) { + agg.uniqueObjects = new HashSet(); +} HashSet uniqueObjs = agg.uniqueObjects; ObjectInspectorObject obj = input instanceof ObjectInspectorObject ? (ObjectInspectorObject)input : @@ -266,7 +269,7 @@ public class GenericUDAFSum extends AbstractGenericUDAFResolver { SumAgg bdAgg = (SumAgg) agg; bdAgg.empty = true; bdAgg.sum = new HiveDecimalWritable(0); - bdAgg.uniqueObjects = new HashSet(); + bdAgg.uniqueObjects = null; } boolean warned = false; @@ -410,7 +413,7 @@ public class GenericUDAFSum extends AbstractGenericUDAFResolver { SumDoubleAgg myagg = (SumDoubleAgg) agg; myagg.empty = true; myagg.sum = 0.0; - myagg.uniqueObjects = new HashSet(); + myagg.uniqueObjects = null; } boolean warned = false; @@ -540,7 +543,7 @@ public class GenericUDAFSum extends AbstractGenericUDAFResolver { SumLongAgg myagg = (SumLongAgg) agg; myagg.empty = true; myagg.sum = 0L; - myagg.uniqueObjects = new HashSet(); + myagg.uniqueObjects = null; } private boolean warned = false;
hive git commit: HIVE-18929: The method humanReadableInt in HiveStringUtils.java has a race condition. (Andrew Sherman, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 5a3f12dd7 -> 1ad48825c HIVE-18929: The method humanReadableInt in HiveStringUtils.java has a race condition. (Andrew Sherman, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1ad48825 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1ad48825 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1ad48825 Branch: refs/heads/master Commit: 1ad48825c01ca326a9b7387d3e044a3a8b2ad81b Parents: 5a3f12d Author: Aihua Xu Authored: Thu Jul 26 16:46:35 2018 -0700 Committer: Aihua Xu Committed: Thu Jul 26 16:46:35 2018 -0700 -- .../hive/common/util/HiveStringUtils.java | 29 1 file changed, 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/1ad48825/common/src/java/org/apache/hive/common/util/HiveStringUtils.java -- diff --git a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java index 6b14ad9..a4923f9 100644 --- a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java +++ b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java @@ -164,35 +164,6 @@ public class HiveStringUtils { return fullHostname; } - private static DecimalFormat oneDecimal = new DecimalFormat("0.0"); - - /** - * Given an integer, return a string that is in an approximate, but human - * readable format. - * It uses the bases 'k', 'm', and 'g' for 1024, 1024**2, and 1024**3. - * @param number the number to format - * @return a human readable form of the integer - */ - public static String humanReadableInt(long number) { -long absNumber = Math.abs(number); -double result = number; -String suffix = ""; -if (absNumber < 1024) { - // since no division has occurred, don't format with a decimal point - return String.valueOf(number); -} else if (absNumber < 1024 * 1024) { - result = number / 1024.0; - suffix = "k"; -} else if (absNumber < 1024 * 1024 * 1024) { - result = number / (1024.0 * 1024); - suffix = "m"; -} else { - result = number / (1024.0 * 1024 * 1024); - suffix = "g"; -} -return oneDecimal.format(result) + suffix; - } - /** * Format a percentage for presentation to the user. * @param done the percentage to format (0.0 to 1.0)
hive git commit: HIVE-20037: Print root cause exception's toString() rather than getMessage() (Aihua Xu, reviewed by Sahil Takiar)
Repository: hive Updated Branches: refs/heads/master a3eacb9dd -> 57dd30441 HIVE-20037: Print root cause exception's toString() rather than getMessage() (Aihua Xu, reviewed by Sahil Takiar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/57dd3044 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/57dd3044 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/57dd3044 Branch: refs/heads/master Commit: 57dd30441a708f9fe653aea1c54df678ed459c34 Parents: a3eacb9 Author: Aihua Xu Authored: Fri Jun 29 14:40:43 2018 -0700 Committer: Aihua Xu Committed: Thu Jul 12 13:38:12 2018 -0700 -- .../hive/ql/exec/spark/session/SparkSessionImpl.java | 10 +++--- .../exec/spark/session/TestSparkSessionManagerImpl.java | 5 +++-- 2 files changed, 6 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/57dd3044/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java index 6e37d93..0f2f031 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java @@ -215,18 +215,14 @@ public class SparkSessionImpl implements SparkSession { sessionId, matchedString.toString()); } else { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId, - getRootCause(oe)); + Throwables.getRootCause(e).toString()); } } e = e.getCause(); } -return new HiveException(oe, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId, getRootCause(oe)); - } - - private String getRootCause(Throwable e) { -Throwable rootCause = Throwables.getRootCause(e); -return rootCause.getClass().getName() + ": " + rootCause.getMessage(); +return new HiveException(oe, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId, +Throwables.getRootCause(oe).toString()); } private boolean matches(String input, String regex, StringBuilder matchedString) { http://git-wip-us.apache.org/repos/asf/hive/blob/57dd3044/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java -- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java index 15756da..6964764 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java @@ -180,8 +180,9 @@ public class TestSparkSessionManagerImpl { "initial executor number 5 must between min executor number10 and max executor number 50"); // Other exceptions which defaults to SPARK_CREATE_CLIENT_ERROR -e = new Exception("Other exception"); -checkHiveException(ss, e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, "Other exception"); +e = new java.lang.NoClassDefFoundError("org/apache/spark/SparkConf"); +checkHiveException(ss, e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, +"java.lang.NoClassDefFoundError: org/apache/spark/SparkConf"); } private void checkHiveException(SparkSessionImpl ss, Throwable e, ErrorMsg expectedErrMsg) {
hive git commit: HIVE-19948: HiveCli is not splitting the command by semicolon properly if quotes are inside the string (Aihua Xu, reviewed by Sahil Takiar)
Repository: hive Updated Branches: refs/heads/master f2c4f3193 -> f37c5de6c HIVE-19948: HiveCli is not splitting the command by semicolon properly if quotes are inside the string (Aihua Xu, reviewed by Sahil Takiar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f37c5de6 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f37c5de6 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f37c5de6 Branch: refs/heads/master Commit: f37c5de6c32b9395d1b34fa3c02ed06d1bfbf6eb Parents: f2c4f31 Author: Aihua Xu Authored: Wed Jun 20 09:01:10 2018 -0700 Committer: Aihua Xu Committed: Mon Jun 25 11:39:55 2018 -0700 -- .../org/apache/hadoop/hive/cli/CliDriver.java | 61 .../hadoop/hive/cli/TestCliDriverMethods.java | 23 2 files changed, 59 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/f37c5de6/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java -- diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index 68741f6..806663d 100644 --- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -416,44 +416,55 @@ public class CliDriver { } } + /** + * Split the line by semicolon by ignoring the ones in the single/double quotes. + * + */ public static List splitSemiColon(String line) { -boolean insideSingleQuote = false; -boolean insideDoubleQuote = false; +boolean inQuotes = false; boolean escape = false; -int beginIndex = 0; + List ret = new ArrayList<>(); + +char quoteChar = '"'; +int beginIndex = 0; for (int index = 0; index < line.length(); index++) { - if (line.charAt(index) == '\'') { -// take a look to see if it is escaped -if (!escape) { - // flip the boolean variable - insideSingleQuote = !insideSingleQuote; -} - } else if (line.charAt(index) == '\"') { -// take a look to see if it is escaped -if (!escape) { - // flip the boolean variable - insideDoubleQuote = !insideDoubleQuote; -} - } else if (line.charAt(index) == ';') { -if (insideSingleQuote || insideDoubleQuote) { - // do not split -} else { - // split, do not include ; itself + char c = line.charAt(index); + switch (c) { + case ';': +if (!inQuotes) { ret.add(line.substring(beginIndex, index)); beginIndex = index + 1; } - } else { -// nothing to do +break; + case '"': + case '\'': +if (!escape) { + if (!inQuotes) { +quoteChar = c; +inQuotes = !inQuotes; + } else { +if (c == quoteChar) { + inQuotes = !inQuotes; +} + } +} +break; + default: +break; } - // set the escape + if (escape) { escape = false; - } else if (line.charAt(index) == '\\') { + } else if (c == '\\') { escape = true; } } -ret.add(line.substring(beginIndex)); + +if (beginIndex < line.length()) { + ret.add(line.substring(beginIndex)); +} + return ret; } http://git-wip-us.apache.org/repos/asf/hive/blob/f37c5de6/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java -- diff --git a/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java b/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java index c06ec3e..8f8f04e 100644 --- a/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java +++ b/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.IDriver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.junit.Test; // Cannot call class TestCliDriver since that's the name of the generated @@ -353,6 +354,28 @@ public class TestCliDriverMethods extends TestCase { } } + @Test + public void testCommandSplits() { +// Test double quote in the string +String cmd1 = "insert into escape1 partition (ds='1', part='\"') values (\"!\")"; +assertEquals(cmd1, CliDriver.splitSemiColon(cmd1).get(0)); +assertEquals(cmd1, CliDriver.splitSemiColon(cmd1 + ";").get(0)); + +// Test escape +String cmd2 = "insert into escape1 partition (ds='1', part='\"\\'') values (\"!\")"; +assertEquals(cmd2, CliD
hive git commit: HIVE-19899: Support stored as JsonFile (Aihua Xu, reviewed by Yongzhi Chen, BELUGA BEHR)
Repository: hive Updated Branches: refs/heads/master 6adab1c2a -> 24e16cc57 HIVE-19899: Support stored as JsonFile (Aihua Xu, reviewed by Yongzhi Chen, BELUGA BEHR) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/24e16cc5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/24e16cc5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/24e16cc5 Branch: refs/heads/master Commit: 24e16cc57293ea6771cd55009f8cfd29870a39ee Parents: 6adab1c Author: Aihua Xu Authored: Thu Jun 14 13:35:49 2018 -0700 Committer: Aihua Xu Committed: Thu Jun 21 14:36:07 2018 -0700 -- .../hcatalog/pig/AbstractHCatStorerTest.java| 2 +- .../pig/TestHCatLoaderComplexSchema.java| 3 ++ .../hive/hcatalog/pig/TestHCatStorer.java | 4 +- .../apache/hadoop/hive/ql/io/IOConstants.java | 1 + .../ql/io/JsonFileStorageFormatDescriptor.java | 51 ...he.hadoop.hive.ql.io.StorageFormatDescriptor | 1 + .../hive/ql/io/TestStorageFormatDescriptor.java | 3 ++ .../test/queries/clientpositive/json_serde1.q | 9 ++-- .../results/clientpositive/json_serde1.q.out| 44 - 9 files changed, 109 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/24e16cc5/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java -- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java index 97277b5..a5cf3a5 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java @@ -54,7 +54,7 @@ import org.slf4j.LoggerFactory; public abstract class AbstractHCatStorerTest extends HCatBaseTest { static Logger LOG = LoggerFactory.getLogger(AbstractHCatStorerTest.class); static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; - String storageFormat; + protected String storageFormat; public AbstractHCatStorerTest() { storageFormat = getStorageFormat(); http://git-wip-us.apache.org/repos/asf/hive/blob/24e16cc5/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java -- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java index 8f06d39..37e670c 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java @@ -75,6 +75,9 @@ public class TestHCatLoaderComplexSchema { put(IOConstants.PARQUETFILE, new HashSet() {{ add("testMapNullKey"); }}); +put(IOConstants.JSONFILE, new HashSet() {{ + add("testMapNullKey"); +}}); }}; private String storageFormat; http://git-wip-us.apache.org/repos/asf/hive/blob/24e16cc5/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java -- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java index 477ea66..cb02139 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java @@ -86,8 +86,6 @@ public class TestHCatStorer extends AbstractHCatStorerTest { } }; - private String storageFormat; - @Parameterized.Parameters public static Collection generateParameters() { return StorageFormats.names(); @@ -99,7 +97,7 @@ public class TestHCatStorer extends AbstractHCatStorerTest { @Override String getStorageFormat() { -return null; +return this.storageFormat; } @Test http://git-wip-us.apache.org/repos/asf/hive/blob/24e16cc5/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java b/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java index f60d296..2be864e 100644 --- a/ql/src/java/org/apach
hive git commit: HIVE-19203: Thread-Safety Issue in HiveMetaStore (Alice Fan, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 230ae3629 -> 6814c8495 HIVE-19203: Thread-Safety Issue in HiveMetaStore (Alice Fan, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6814c849 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6814c849 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6814c849 Branch: refs/heads/master Commit: 6814c84951b41486f6c8af07b0a79f8f0938b636 Parents: 230ae36 Author: Aihua Xu Authored: Tue Jun 12 15:16:38 2018 -0700 Committer: Aihua Xu Committed: Tue Jun 12 15:16:38 2018 -0700 -- .../java/org/apache/hadoop/hive/metastore/HiveMetaStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/6814c849/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java -- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 8d7b0c3..7dbdba8 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -350,11 +350,11 @@ public class HiveMetaStore extends ThriftHiveMetastore { return null; } -private static int nextSerialNum = 0; +private static AtomicInteger nextSerialNum = new AtomicInteger(); private static ThreadLocal threadLocalId = new ThreadLocal() { @Override protected Integer initialValue() { -return nextSerialNum++; +return nextSerialNum.getAndIncrement(); } };
hive git commit: HIVE-19053: RemoteSparkJobStatus#getSparkJobInfo treats all exceptions as timeout errors (Aihua Xu, reviewed by Sahil Takiar)
Repository: hive Updated Branches: refs/heads/master 148635cb8 -> 913baef82 HIVE-19053: RemoteSparkJobStatus#getSparkJobInfo treats all exceptions as timeout errors (Aihua Xu, reviewed by Sahil Takiar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/913baef8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/913baef8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/913baef8 Branch: refs/heads/master Commit: 913baef8279f6682f7b209a1f4e7c445a85f17e9 Parents: 148635c Author: Aihua Xu Authored: Tue Jun 5 13:47:12 2018 -0700 Committer: Aihua Xu Committed: Fri Jun 8 11:58:34 2018 -0700 -- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java| 2 ++ .../exec/spark/status/impl/RemoteSparkJobStatus.java | 13 ++--- 2 files changed, 12 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/913baef8/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 8baf309..bc2cffa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -585,6 +585,8 @@ public enum ErrorMsg { "Cannot create Spark client on a closed session {0}", true), SPARK_JOB_INTERRUPTED(30044, "Spark job was interrupted while executing"), + SPARK_GET_JOB_INFO_INTERRUPTED(30045, "Spark job was interrupted while getting job info"), + SPARK_GET_JOB_INFO_EXECUTIONERROR(30046, "Spark job failed in execution while getting job info due to exception {0}"), //== 4 range starts here // http://git-wip-us.apache.org/repos/asf/hive/blob/913baef8/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java index d2e28b0..832832b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java @@ -25,6 +25,8 @@ import org.apache.hadoop.hive.ql.exec.spark.status.SparkStage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Throwables; + import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.spark.Statistic.SparkStatistics; import org.apache.hadoop.hive.ql.exec.spark.Statistic.SparkStatisticsBuilder; @@ -37,7 +39,6 @@ import org.apache.hive.spark.client.JobContext; import org.apache.hive.spark.client.JobHandle; import org.apache.hive.spark.client.SparkClient; import org.apache.hive.spark.counter.SparkCounters; - import org.apache.spark.JobExecutionStatus; import org.apache.spark.SparkJobInfo; import org.apache.spark.SparkStageInfo; @@ -47,8 +48,10 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; /** * Used with remove spark client. @@ -198,10 +201,14 @@ public class RemoteSparkJobStatus implements SparkJobStatus { new GetJobInfoJob(jobHandle.getClientJobId(), sparkJobId)); try { return getJobInfo.get(sparkClientTimeoutInSeconds, TimeUnit.SECONDS); -} catch (Exception e) { - LOG.warn("Failed to get job info.", e); +} catch (TimeoutException e) { throw new HiveException(e, ErrorMsg.SPARK_GET_JOB_INFO_TIMEOUT, Long.toString(sparkClientTimeoutInSeconds)); +} catch (InterruptedException e) { + throw new HiveException(e, ErrorMsg.SPARK_GET_JOB_INFO_INTERRUPTED); +} catch (ExecutionException e) { + throw new HiveException(e, ErrorMsg.SPARK_GET_JOB_INFO_EXECUTIONERROR, + Throwables.getRootCause(e).getMessage()); } }
hive git commit: HIVE-18766: Race condition during shutdown of RemoteDriver, error messages aren't always sent (Aihua Xu, reviewed by Sahil Takiar)
Repository: hive Updated Branches: refs/heads/master 77c145043 -> aae62d871 HIVE-18766: Race condition during shutdown of RemoteDriver, error messages aren't always sent (Aihua Xu, reviewed by Sahil Takiar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aae62d87 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aae62d87 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aae62d87 Branch: refs/heads/master Commit: aae62d871bd3bf61281b03e2ef183b214e610cd5 Parents: 77c1450 Author: Aihua Xu Authored: Tue Jun 5 13:16:54 2018 -0700 Committer: Aihua Xu Committed: Wed Jun 6 14:14:39 2018 -0700 -- .../org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../apache/hive/spark/client/RemoteDriver.java | 44 +--- .../hive/spark/client/rpc/RpcConfiguration.java | 8 3 files changed, 37 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/aae62d87/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java -- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index cd425aa..dd42fd1 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4206,7 +4206,7 @@ public class HiveConf extends Configuration { "in shuffle. This should result in less shuffled data."), SPARK_CLIENT_FUTURE_TIMEOUT("hive.spark.client.future.timeout", "60s", new TimeValidator(TimeUnit.SECONDS), - "Timeout for requests from Hive client to remote Spark driver."), + "Timeout for requests between Hive client and remote Spark driver."), SPARK_JOB_MONITOR_TIMEOUT("hive.spark.job.monitor.timeout", "60s", new TimeValidator(TimeUnit.SECONDS), "Timeout for job monitor to get Spark job state."), http://git-wip-us.apache.org/repos/asf/hive/blob/aae62d87/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java -- diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java b/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java index caa850c..8130860 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java @@ -35,6 +35,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.FileUtils; @@ -92,6 +94,8 @@ public class RemoteDriver { public static final String REMOTE_DRIVER_PORT_CONF = "--remote-port"; public static final String REMOTE_DRIVER_CONF = "--remote-driver-conf"; + private final long futureTimeout; // Rpc call timeout in milliseconds + private RemoteDriver(String[] args) throws Exception { this.activeJobs = Maps.newConcurrentMap(); this.jcLock = new Object(); @@ -135,7 +139,9 @@ public class RemoteDriver { String secret = mapConf.get(SparkClientFactory.CONF_KEY_SECRET); Preconditions.checkArgument(secret != null, "No secret provided."); -int threadCount = new RpcConfiguration(mapConf).getRpcThreadCount(); +RpcConfiguration rpcConf = new RpcConfiguration(mapConf); +futureTimeout = rpcConf.getFutureTimeoutMs(); +int threadCount = rpcConf.getRpcThreadCount(); this.egroup = new NioEventLoopGroup( threadCount, new ThreadFactoryBuilder() @@ -232,13 +238,19 @@ public class RemoteDriver { for (JobWrapper job : activeJobs.values()) { cancelJob(job); } + if (error != null) { -protocol.sendError(error); +try { + protocol.sendError(error).get(futureTimeout, TimeUnit.MILLISECONDS); +} catch(InterruptedException|ExecutionException|TimeoutException e) { + LOG.warn("Failed to send out the error during RemoteDriver shutdown", e); +} } if (jc != null) { jc.stop(); } clientRpc.close(); + egroup.shutdownGracefully(); synchronized (shutdownLock) { shutdownLock.notifyAll(); @@ -265,34 +277,35 @@ public class RemoteDriver { private class DriverProtocol extends BaseProtocol { -void sendError(Throwable error) { +Future sendError(Throwable error) { LOG.debug("Send error to Client: {}", Throwables.getStackTraceAsString(error)); - clientRpc.call(new Error(Throwables.getStackTr
hive git commit: HIVE-19662: Upgrade Avro to 1.8.2 (Janaki Lahorani, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 15bf4eb98 -> 664de83d2 HIVE-19662: Upgrade Avro to 1.8.2 (Janaki Lahorani, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/664de83d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/664de83d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/664de83d Branch: refs/heads/master Commit: 664de83d24cbddc13abe4087328fc0fbd4268503 Parents: 15bf4eb Author: Aihua Xu Authored: Wed May 30 11:20:54 2018 -0700 Committer: Aihua Xu Committed: Wed May 30 12:04:07 2018 -0700 -- hbase-handler/pom.xml | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/664de83d/hbase-handler/pom.xml -- diff --git a/hbase-handler/pom.xml b/hbase-handler/pom.xml index be8e0b9..06939a4 100644 --- a/hbase-handler/pom.xml +++ b/hbase-handler/pom.xml @@ -227,7 +227,7 @@ org.apache.avro avro - 1.7.6 + ${avro.version} @@ -249,7 +249,7 @@ org.apache.avro avro-maven-plugin -1.7.6 +${avro.version} generate-test-sources http://git-wip-us.apache.org/repos/asf/hive/blob/664de83d/pom.xml -- diff --git a/pom.xml b/pom.xml index db847a1..462b62f 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ 0.8.0 1.11.0 -1.7.7 +1.8.2 0.8.0.RELEASE 1.16.0 4.2.4
hive git commit: HIVE-19424: NPE In MetaDataFormatters (Alice Fan, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 6875c7655 -> 29f57fc73 HIVE-19424: NPE In MetaDataFormatters (Alice Fan, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/29f57fc7 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/29f57fc7 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/29f57fc7 Branch: refs/heads/master Commit: 29f57fc73ef46dc4b5b6ea7c74e493e3cb3e2f7f Parents: 6875c76 Author: Aihua Xu Authored: Wed May 16 11:27:53 2018 -0700 Committer: Aihua Xu Committed: Wed May 16 14:06:37 2018 -0700 -- .../hive/ql/metadata/formatting/JsonMetaDataFormatter.java | 8 .../hive/ql/metadata/formatting/TextMetaDataFormatter.java | 8 2 files changed, 8 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/29f57fc7/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java index e7b5af6..df0a237 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java @@ -204,15 +204,15 @@ public class JsonMetaDataFormatter implements MetaDataFormatter { if (par.getLocation() != null) { tblLoc = par.getDataLocation().toString(); } -inputFormattCls = par.getInputFormatClass().getName(); -outputFormattCls = par.getOutputFormatClass().getName(); +inputFormattCls = par.getInputFormatClass() == null ? null : par.getInputFormatClass().getName(); +outputFormattCls = par.getOutputFormatClass() == null ? null : par.getOutputFormatClass().getName(); } } else { if (tbl.getPath() != null) { tblLoc = tbl.getDataLocation().toString(); } - inputFormattCls = tbl.getInputFormatClass().getName(); - outputFormattCls = tbl.getOutputFormatClass().getName(); + inputFormattCls = tbl.getInputFormatClass() == null ? null : tbl.getInputFormatClass().getName(); + outputFormattCls = tbl.getOutputFormatClass() == null ? null : tbl.getOutputFormatClass().getName(); } MapBuilder builder = MapBuilder.create(); http://git-wip-us.apache.org/repos/asf/hive/blob/29f57fc7/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java index 2529923..326cbed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java @@ -291,15 +291,15 @@ class TextMetaDataFormatter implements MetaDataFormatter { if (par.getLocation() != null) { tblLoc = par.getDataLocation().toString(); } -inputFormattCls = par.getInputFormatClass().getName(); -outputFormattCls = par.getOutputFormatClass().getName(); +inputFormattCls = par.getInputFormatClass() == null ? null : par.getInputFormatClass().getName(); +outputFormattCls = par.getOutputFormatClass() == null ? null : par.getOutputFormatClass().getName(); } } else { if (tbl.getPath() != null) { tblLoc = tbl.getDataLocation().toString(); } - inputFormattCls = tbl.getInputFormatClass().getName(); - outputFormattCls = tbl.getOutputFormatClass().getName(); + inputFormattCls = tbl.getInputFormatClass() == null ? null : tbl.getInputFormatClass().getName(); + outputFormattCls = tbl.getOutputFormatClass() == null ? null : tbl.getOutputFormatClass().getName(); } String owner = tbl.getOwner();
hive git commit: HIVE-19496: Check untar folder (Aihua Xu, reviewed by Sahil Takiar)
Repository: hive Updated Branches: refs/heads/branch-3 f1f265046 -> 3c44a3820 HIVE-19496: Check untar folder (Aihua Xu, reviewed by Sahil Takiar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c44a382 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c44a382 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c44a382 Branch: refs/heads/branch-3 Commit: 3c44a3820424de3dc07a80714732b548ae147020 Parents: f1f2650 Author: Aihua Xu Authored: Thu May 10 14:51:57 2018 -0700 Committer: Aihua Xu Committed: Wed May 16 10:19:08 2018 -0700 -- .../src/java/org/apache/hadoop/hive/common/CompressionUtils.java | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/3c44a382/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java -- diff --git a/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java b/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java index 681c506..d98632e 100644 --- a/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java @@ -159,6 +159,10 @@ public class CompressionUtils { TarArchiveEntry entry = null; while ((entry = (TarArchiveEntry) debInputStream.getNextEntry()) != null) { final File outputFile = new File(outputDir, entry.getName()); + if (!outputFile.toPath().toAbsolutePath().normalize() + .startsWith(outputDir.toPath().toAbsolutePath().normalize())) { +throw new IOException("Untarred file is not under the output directory"); + } if (entry.isDirectory()) { if (flatten) { // no sub-directories
hive git commit: HIVE-19496: Check untar folder (Aihua Xu, reviewed by Sahil Takiar)
Repository: hive Updated Branches: refs/heads/master ff446b779 -> 6e6b0cb7b HIVE-19496: Check untar folder (Aihua Xu, reviewed by Sahil Takiar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6e6b0cb7 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6e6b0cb7 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6e6b0cb7 Branch: refs/heads/master Commit: 6e6b0cb7b1950e6b0e4a19b0f9e2d185031dd83f Parents: ff446b7 Author: Aihua Xu Authored: Thu May 10 14:51:57 2018 -0700 Committer: Aihua Xu Committed: Tue May 15 10:11:19 2018 -0700 -- .../src/java/org/apache/hadoop/hive/common/CompressionUtils.java | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/6e6b0cb7/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java -- diff --git a/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java b/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java index 681c506..d98632e 100644 --- a/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java @@ -159,6 +159,10 @@ public class CompressionUtils { TarArchiveEntry entry = null; while ((entry = (TarArchiveEntry) debInputStream.getNextEntry()) != null) { final File outputFile = new File(outputDir, entry.getName()); + if (!outputFile.toPath().toAbsolutePath().normalize() + .startsWith(outputDir.toPath().toAbsolutePath().normalize())) { +throw new IOException("Untarred file is not under the output directory"); + } if (entry.isDirectory()) { if (flatten) { // no sub-directories
hive git commit: HIVE-19228: Remove commons-httpclient 3.x usage (Janaki Lahorani reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master e1e497574 -> f2172cdbc HIVE-19228: Remove commons-httpclient 3.x usage (Janaki Lahorani reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f2172cdb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f2172cdb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f2172cdb Branch: refs/heads/master Commit: f2172cdbc3af92e18db68c3daac44a73e92eaf48 Parents: e1e4975 Author: Aihua Xu Authored: Wed May 9 10:56:32 2018 -0700 Committer: Aihua Xu Committed: Wed May 9 11:27:45 2018 -0700 -- .../apache/hive/jdbc/TestActivePassiveHA.java | 99 +--- pom.xml | 6 -- ql/pom.xml | 15 --- .../hive/ql/parse/LoadSemanticAnalyzer.java | 16 +++- .../apache/hive/service/server/HiveServer2.java | 52 ++ 5 files changed, 111 insertions(+), 77 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/f2172cdb/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java index c55271f..4055f13 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java @@ -36,11 +36,6 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethodBase; -import org.apache.commons.httpclient.methods.DeleteMethod; -import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.httpclient.methods.OptionsMethod; import org.apache.curator.test.TestingServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -54,9 +49,22 @@ import org.apache.hive.service.server.HS2ActivePassiveHARegistryClient; import org.apache.hive.service.server.HiveServer2Instance; import org.apache.hive.service.server.TestHS2HttpServerPam; import org.apache.hive.service.servlet.HS2Peers; +import org.apache.http.Header; import org.apache.http.HttpException; import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpOptions; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.StatusLine; +import org.apache.http.util.EntityUtils; import org.codehaus.jackson.map.ObjectMapper; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.util.B64Code; +import org.eclipse.jetty.util.StringUtil; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -406,7 +414,7 @@ public class TestActivePassiveHA { assertEquals("true", sendGet(url1, true)); // trigger failover on miniHS2_1 without authorization header - assertEquals("Unauthorized", sendDelete(url1, false)); + assertTrue(sendDelete(url1, false).contains("Unauthorized")); assertTrue(sendDelete(url1, true).contains("Failover successful!")); assertEquals(true, miniHS2_1.getNotLeaderTestFuture().get()); assertEquals(false, miniHS2_1.isLeader()); @@ -541,56 +549,79 @@ public class TestActivePassiveHA { } private String sendGet(String url, boolean enableAuth) throws Exception { -return sendAuthMethod(new GetMethod(url), enableAuth, false); +return sendAuthMethod(new HttpGet(url), enableAuth, false); } private String sendGet(String url, boolean enableAuth, boolean enableCORS) throws Exception { -return sendAuthMethod(new GetMethod(url), enableAuth, enableCORS); +return sendAuthMethod(new HttpGet(url), enableAuth, enableCORS); } private String sendDelete(String url, boolean enableAuth) throws Exception { -return sendAuthMethod(new DeleteMethod(url), enableAuth, false); +return sendAuthMethod(new HttpDelete(url), enableAuth, false); } private String sendDelete(String url, boolean enableAuth, boolean enableCORS) throws Exception { -return sendAuthMethod(new DeleteMethod(url), enableAuth, enableCORS); +return sendAuthMethod(new HttpDelete(url), enableAuth, enableCORS); } - private String sendAuthMethod(HttpMethodBase method, boolean enableAuth, boolean enableCORS) throws Exception { -HttpClient client = new HttpClient(); -try { -
hive git commit: HIVE-18915: Better client logging when a HoS session can't be opened (Aihua Xu, reviewed by Sahil Takiar)
Repository: hive Updated Branches: refs/heads/master f8bc4868e -> 3c4a9c6bc HIVE-18915: Better client logging when a HoS session can't be opened (Aihua Xu, reviewed by Sahil Takiar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c4a9c6b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c4a9c6b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c4a9c6b Branch: refs/heads/master Commit: 3c4a9c6bc37dfc870800523a6d26f667d0b43818 Parents: f8bc486 Author: Aihua Xu Authored: Thu Apr 12 17:12:56 2018 -0700 Committer: Aihua Xu Committed: Mon Apr 30 16:35:41 2018 -0700 -- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 2 +- .../ql/exec/spark/session/SparkSessionImpl.java | 30 +--- .../session/TestSparkSessionManagerImpl.java| 6 ++-- 3 files changed, 17 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/3c4a9c6b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 7d33fa3..94dd636 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -576,7 +576,7 @@ public enum ErrorMsg { SPARK_CREATE_CLIENT_INTERRUPTED(30040, "Interrupted while creating Spark client for session {0}", true), SPARK_CREATE_CLIENT_ERROR(30041, - "Failed to create Spark client for Spark session {0}", true), + "Failed to create Spark client for Spark session {0}: {1}", true), SPARK_CREATE_CLIENT_INVALID_RESOURCE_REQUEST(30042, "Failed to create Spark client due to invalid resource request: {0}", true), SPARK_CREATE_CLIENT_CLOSED_SESSION(30043, http://git-wip-us.apache.org/repos/asf/hive/blob/3c4a9c6b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java index 2d5d03e..189de19 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java @@ -71,7 +71,6 @@ public class SparkSessionImpl implements SparkSession { private HiveSparkClient hiveSparkClient; private Path scratchDir; private final Object dirLock = new Object(); - private String matchedString = null; public SparkSessionImpl() { sessionId = makeSessionId(); @@ -195,6 +194,7 @@ public class SparkSessionImpl implements SparkSession { @VisibleForTesting HiveException getHiveException(Throwable e) { Throwable oe = e; +StringBuilder matchedString = new StringBuilder(); while (e != null) { if (e instanceof TimeoutException) { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_TIMEOUT); @@ -202,31 +202,26 @@ public class SparkSessionImpl implements SparkSession { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INTERRUPTED, sessionId); } else if (e instanceof RuntimeException) { String sts = Throwables.getStackTraceAsString(e); -if (matches(sts, AM_TIMEOUT_ERR)) { +if (matches(sts, AM_TIMEOUT_ERR, matchedString)) { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_TIMEOUT); -} else if (matches(sts, UNKNOWN_QUEUE_ERR) || matches(sts, STOPPED_QUEUE_ERR)) { - return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_QUEUE, matchedString); -} else if (matches(sts, FULL_QUEUE_ERR)) { - return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_QUEUE_FULL, matchedString); -} else if (matches(sts, INVALILD_MEM_ERR) || matches(sts, INVALID_CORE_ERR)) { +} else if (matches(sts, UNKNOWN_QUEUE_ERR, matchedString) || matches(sts, STOPPED_QUEUE_ERR, matchedString)) { + return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_QUEUE, matchedString.toString()); +} else if (matches(sts, FULL_QUEUE_ERR, matchedString)) { + return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_QUEUE_FULL, matchedString.toString()); +} else if (matches(sts, INVALILD_MEM_ERR, matchedString) || matches(sts, INVALID_CORE_ERR, matchedString)) { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_RESOURCE_REQUEST, - matchedString); + matchedString.toString()); } else { - return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId); + return new HiveExcepti
[1/2] hive git commit: HIVE-19204: Detailed errors from some tasks are not displayed to the client because the tasks don't set exception when they fail (Aihua Xu, reviewed by Sahil Takiar)
Repository: hive Updated Branches: refs/heads/master f94ae7fec -> f30efbebf HIVE-19204: Detailed errors from some tasks are not displayed to the client because the tasks don't set exception when they fail (Aihua Xu, reviewed by Sahil Takiar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/11b0d857 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/11b0d857 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/11b0d857 Branch: refs/heads/master Commit: 11b0d85786cd58469d5662c3027e9389cff07710 Parents: f94ae7f Author: Aihua Xu Authored: Mon Apr 16 10:36:02 2018 -0700 Committer: Aihua Xu Committed: Wed Apr 25 16:09:42 2018 -0700 -- .../java/org/apache/hadoop/hive/ql/Driver.java | 6 - .../hive/ql/exec/ColumnStatsUpdateTask.java | 1 + .../hive/ql/exec/ExplainSQRewriteTask.java | 8 +++--- .../apache/hadoop/hive/ql/exec/ExplainTask.java | 5 ++-- .../hive/ql/exec/MaterializedViewTask.java | 1 + .../hadoop/hive/ql/exec/ReplCopyTask.java | 4 +-- .../apache/hadoop/hive/ql/exec/StatsTask.java | 1 + .../hadoop/hive/ql/exec/mr/ExecDriver.java | 4 +-- .../io/rcfile/truncate/ColumnTruncateTask.java | 26 +++- .../ql/reexec/ReExecutionOverlayPlugin.java | 2 +- 10 files changed, 29 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/11b0d857/ql/src/java/org/apache/hadoop/hive/ql/Driver.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 4e8dbe2..f83bdaf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -2389,7 +2389,11 @@ public class Driver implements IDriver { if(downstreamError != null) { //here we assume that upstream code may have parametrized the msg from ErrorMsg //so we want to keep it - errorMessage += ". " + downstreamError.getMessage(); + if (downstreamError.getMessage() != null) { +errorMessage += ". " + downstreamError.getMessage(); + } else { +errorMessage += ". " + org.apache.hadoop.util.StringUtils.stringifyException(downstreamError); + } } else { ErrorMsg em = ErrorMsg.getErrorMsg(exitVal); http://git-wip-us.apache.org/repos/asf/hive/blob/11b0d857/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java index 207b66f..a53ff5a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java @@ -300,6 +300,7 @@ public class ColumnStatsUpdateTask extends Task { Hive db = getHive(); return persistColumnStats(db); } catch (Exception e) { + setException(e); LOG.info("Failed to persist stats in metastore", e); } return 1; http://git-wip-us.apache.org/repos/asf/hive/blob/11b0d857/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainSQRewriteTask.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainSQRewriteTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainSQRewriteTask.java index 80d54bf..1f9e9aa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainSQRewriteTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainSQRewriteTask.java @@ -38,11 +38,13 @@ import org.apache.hadoop.hive.ql.parse.SubQueryDiagnostic; import org.apache.hadoop.hive.ql.plan.ExplainSQRewriteWork; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ExplainSQRewriteTask extends Task implements Serializable { private static final long serialVersionUID = 1L; + private final Logger LOG = LoggerFactory.getLogger(this.getClass().getName()); @Override public StageType getType() { @@ -76,8 +78,8 @@ public class ExplainSQRewriteTask extends Task implements return (0); } catch (Exception e) { - console.printError("Failed with exception " + e.getMessage(), - "\n" + StringUtils.stringifyException(e)); + setException(e); + LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); return (1); } finally { http://git-wip-us.apache.org/repos/asf/hive/blob/11b0d857/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java ---
[2/2] hive git commit: HIVE-18986: Table rename will run java.lang.StackOverflowError in dataNucleus if the table contains large number of columns (Aihua Xu, reviewed by Yongzhi Chen)
HIVE-18986: Table rename will run java.lang.StackOverflowError in dataNucleus if the table contains large number of columns (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f30efbeb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f30efbeb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f30efbeb Branch: refs/heads/master Commit: f30efbebf2ff85c55a5d9e3e2f86e0a51341df78 Parents: 11b0d85 Author: Aihua Xu Authored: Wed Apr 18 17:05:08 2018 -0700 Committer: Aihua Xu Committed: Wed Apr 25 16:10:30 2018 -0700 -- .../queries/clientpositive/alter_rename_table.q | 12 ++- .../clientpositive/alter_rename_table.q.out | 88 .../apache/hadoop/hive/metastore/Batchable.java | 86 +++ .../hive/metastore/MetaStoreDirectSql.java | 61 ++ .../hadoop/hive/metastore/ObjectStore.java | 45 ++ .../hive/metastore/conf/MetastoreConf.java | 5 ++ 6 files changed, 227 insertions(+), 70 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/f30efbeb/ql/src/test/queries/clientpositive/alter_rename_table.q -- diff --git a/ql/src/test/queries/clientpositive/alter_rename_table.q b/ql/src/test/queries/clientpositive/alter_rename_table.q index 53fb230..bcf6ad5 100644 --- a/ql/src/test/queries/clientpositive/alter_rename_table.q +++ b/ql/src/test/queries/clientpositive/alter_rename_table.q @@ -36,4 +36,14 @@ create table source.src1 like default.src; load data local inpath '../../data/files/kv1.txt' overwrite into table source.src; ALTER TABLE source.src RENAME TO target.src1; -select * from target.src1 tablesample (10 rows); \ No newline at end of file +select * from target.src1 tablesample (10 rows); + +set metastore.rawstore.batch.size=1; +set metastore.try.direct.sql=false; + +create table source.src2 like default.src; +load data local inpath '../../data/files/kv1.txt' overwrite into table source.src2; +ANALYZE TABlE source.src2 COMPUTE STATISTICS FOR COLUMNS; +ALTER TABLE source.src2 RENAME TO target.src3; +DESC FORMATTED target.src3; +select * from target.src3 tablesample (10 rows); http://git-wip-us.apache.org/repos/asf/hive/blob/f30efbeb/ql/src/test/results/clientpositive/alter_rename_table.q.out -- diff --git a/ql/src/test/results/clientpositive/alter_rename_table.q.out b/ql/src/test/results/clientpositive/alter_rename_table.q.out index 732d8a2..9ac8fd2 100644 --- a/ql/src/test/results/clientpositive/alter_rename_table.q.out +++ b/ql/src/test/results/clientpositive/alter_rename_table.q.out @@ -261,3 +261,91 @@ POSTHOOK: Input: target@src1 278val_278 98 val_98 484val_484 +PREHOOK: query: create table source.src2 like default.src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:source +PREHOOK: Output: source@src2 +POSTHOOK: query: create table source.src2 like default.src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:source +POSTHOOK: Output: source@src2 +PREHOOK: query: load data local inpath '../../data/files/kv1.txt' overwrite into table source.src2 +PREHOOK: type: LOAD + A masked pattern was here +PREHOOK: Output: source@src2 +POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' overwrite into table source.src2 +POSTHOOK: type: LOAD + A masked pattern was here +POSTHOOK: Output: source@src2 +PREHOOK: query: ANALYZE TABlE source.src2 COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: QUERY +PREHOOK: Input: source@src2 + A masked pattern was here +PREHOOK: Output: source@src2 +POSTHOOK: query: ANALYZE TABlE source.src2 COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: QUERY +POSTHOOK: Input: source@src2 + A masked pattern was here +POSTHOOK: Output: source@src2 +PREHOOK: query: ALTER TABLE source.src2 RENAME TO target.src3 +PREHOOK: type: ALTERTABLE_RENAME +PREHOOK: Input: source@src2 +PREHOOK: Output: source@src2 +POSTHOOK: query: ALTER TABLE source.src2 RENAME TO target.src3 +POSTHOOK: type: ALTERTABLE_RENAME +POSTHOOK: Input: source@src2 +POSTHOOK: Output: source@src2 +POSTHOOK: Output: target@src3 +PREHOOK: query: DESC FORMATTED target.src3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: target@src3 +POSTHOOK: query: DESC FORMATTED target.src3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: target@src3 +# col_name data_type comment +keystring default +value string default + +# Detailed Table Information +Database: target + A masked pattern was here +Retention:
hive git commit: HIVE-17429: Hive JDBC doesn't return rows when querying Impala (Zach Amsden, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/branch-2.3 f78128a71 -> 1ca13b3ce HIVE-17429: Hive JDBC doesn't return rows when querying Impala (Zach Amsden, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1ca13b3c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1ca13b3c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1ca13b3c Branch: refs/heads/branch-2.3 Commit: 1ca13b3cec97c4114090687079b6efbffeb498c3 Parents: f78128a Author: Aihua Xu Authored: Thu Sep 7 17:17:14 2017 -0700 Committer: Aihua Xu Committed: Thu Apr 12 15:33:16 2018 -0700 -- jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/1ca13b3c/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java -- diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java index c385e2c..4c98517 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java @@ -254,7 +254,7 @@ public class HiveStatement implements java.sql.Statement { TGetOperationStatusResp status = waitForOperationToComplete(); // The query should be completed by now -if (!status.isHasResultSet()) { +if (!status.isHasResultSet() && !stmtHandle.isHasResultSet()) { return false; } resultSet = new HiveQueryResultSet.Builder(this).setClient(client).setSessionHandle(sessHandle)
hive git commit: HIVE-17429: Hive JDBC doesn't return rows when querying Impala (Zach Amsden, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/branch-2.2 9e10b88c3 -> 1846c5a73 HIVE-17429: Hive JDBC doesn't return rows when querying Impala (Zach Amsden, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1846c5a7 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1846c5a7 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1846c5a7 Branch: refs/heads/branch-2.2 Commit: 1846c5a737ee32b49dbeb6dd58ca7968f68fa31a Parents: 9e10b88 Author: Aihua Xu Authored: Thu Sep 7 17:17:14 2017 -0700 Committer: Aihua Xu Committed: Thu Apr 12 15:26:30 2018 -0700 -- jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/1846c5a7/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java -- diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java index ed0ed08..d54c964 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java @@ -254,7 +254,7 @@ public class HiveStatement implements java.sql.Statement { TGetOperationStatusResp status = waitForOperationToComplete(); // The query should be completed by now -if (!status.isHasResultSet()) { +if (!status.isHasResultSet() && !stmtHandle.isHasResultSet()) { return false; } resultSet = new HiveQueryResultSet.Builder(this).setClient(client).setSessionHandle(sessHandle)
hive git commit: HIVE-19018: beeline -e now requires semicolon even when used with query from command line (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master eea736134 -> 52290e72b HIVE-19018: beeline -e now requires semicolon even when used with query from command line (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/52290e72 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/52290e72 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/52290e72 Branch: refs/heads/master Commit: 52290e72b3029eddc17f0a0482fd6c1a05840bb0 Parents: eea7361 Author: Aihua Xu Authored: Wed Mar 21 17:36:50 2018 -0700 Committer: Aihua Xu Committed: Fri Mar 30 10:59:02 2018 -0700 -- .../src/java/org/apache/hive/beeline/BeeLine.java | 2 ++ .../src/java/org/apache/hive/beeline/Commands.java | 1 - .../test/org/apache/hive/beeline/TestCommands.java | 16 3 files changed, 18 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/52290e72/beeline/src/java/org/apache/hive/beeline/BeeLine.java -- diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 402fadd..c6d009c 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -795,6 +795,8 @@ public class BeeLine implements Closeable { int code = 0; if (cl.getOptionValues('e') != null) { commands = Arrays.asList(cl.getOptionValues('e')); + opts.setAllowMultiLineCommand(false); //When using -e, command is always a single line + } if (!commands.isEmpty() && getOpts().getScriptFile() != null) { http://git-wip-us.apache.org/repos/asf/hive/blob/52290e72/beeline/src/java/org/apache/hive/beeline/Commands.java -- diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index d5806a4..e46c0cf 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -1072,7 +1072,6 @@ public class Commands { * Check if the input line is a multi-line command which needs to read further */ public String handleMultiLineCmd(String line) throws IOException { -//When using -e, console reader is not initialized and command is always a single line int[] startQuote = {-1}; line = HiveStringUtils.removeComments(line, startQuote); while (isMultiLine(line) && beeLine.getOpts().isAllowMultiLineCommand()) { http://git-wip-us.apache.org/repos/asf/hive/blob/52290e72/beeline/src/test/org/apache/hive/beeline/TestCommands.java -- diff --git a/beeline/src/test/org/apache/hive/beeline/TestCommands.java b/beeline/src/test/org/apache/hive/beeline/TestCommands.java index 8a51f98..567ca25 100644 --- a/beeline/src/test/org/apache/hive/beeline/TestCommands.java +++ b/beeline/src/test/org/apache/hive/beeline/TestCommands.java @@ -23,6 +23,8 @@ import org.junit.Test; import static org.apache.hive.common.util.HiveStringUtils.removeComments; import static org.junit.Assert.assertEquals; +import java.io.IOException; + public class TestCommands { @Test @@ -43,5 +45,19 @@ public class TestCommands { assertEquals("'show --comments tables'", removeComments("'show --comments tables' --comments",escape)); assertEquals("'\"show --comments tables\"'", removeComments("'\"show --comments tables\"' --comments",escape)); } + + /** + * Test the commands directly call from beeline. + * @throws IOException + */ + @Test + public void testBeelineCommands() throws IOException { + // avoid System.exit() call in beeline which causes JVM to exit and fails the test +System.setProperty(BeeLineOpts.PROPERTY_NAME_EXIT, "true"); +// Verify the command without ';' at the end also works fine +BeeLine.mainWithInputRedirection(new String[] {"-u", "jdbc:hive2://", "-e", "select 3"}, null); +BeeLine.mainWithInputRedirection( +new String[] {"-u", "jdbc:hive2://", "-e", "create table t1(x int); show tables"}, null); + } }
hive git commit: HIVE-18433: Upgrade version of com.fasterxml.jackson (Janaki Lahorani, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 680a58476 -> 245c39b4c HIVE-18433: Upgrade version of com.fasterxml.jackson (Janaki Lahorani, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/245c39b4 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/245c39b4 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/245c39b4 Branch: refs/heads/master Commit: 245c39b4c8f711fbc1c9c00df013e4c7fcbdc0a2 Parents: 680a584 Author: Aihua Xu Authored: Tue Mar 27 13:51:02 2018 -0700 Committer: Aihua Xu Committed: Tue Mar 27 13:51:02 2018 -0700 -- common/pom.xml | 2 +- druid-handler/pom.xml | 8 + .../hive/druid/DruidStorageHandlerUtils.java| 2 +- .../TestHiveDruidQueryBasedInputFormat.java | 6 ++-- hcatalog/core/pom.xml | 14 ++-- hcatalog/server-extensions/pom.xml | 14 ++-- hcatalog/webhcat/svr/pom.xml| 15 ++--- itests/hive-blobstore/pom.xml | 6 ++-- itests/qtest-druid/pom.xml | 4 +-- pom.xml | 35 ++-- ql/pom.xml | 21 +++- spark-client/pom.xml| 26 ++- standalone-metastore/pom.xml| 4 +-- testutils/ptest2/pom.xml| 17 -- .../hive/ptest/api/client/PTestClient.java | 2 +- .../hive/ptest/execution/JIRAService.java | 8 ++--- 16 files changed, 128 insertions(+), 56 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/245c39b4/common/pom.xml -- diff --git a/common/pom.xml b/common/pom.xml index 4da46f2..9393d87 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -250,7 +250,7 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.new.version} + ${jackson.version} com.github.joshelser http://git-wip-us.apache.org/repos/asf/hive/blob/245c39b4/druid-handler/pom.xml -- diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml index b53ddb4..93eb91e 100644 --- a/druid-handler/pom.xml +++ b/druid-handler/pom.xml @@ -37,6 +37,14 @@ + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + + + com.fasterxml.jackson.core + jackson-databind + + org.apache.hive hive-exec ${project.version} http://git-wip-us.apache.org/repos/asf/hive/blob/245c39b4/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java -- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java index a71a3af..233b288 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java @@ -280,7 +280,7 @@ public final class DruidStorageHandlerUtils { } for (FileStatus fileStatus : fss) { final DataSegment segment = JSON_MAPPER - .readValue(fs.open(fileStatus.getPath()), DataSegment.class); + .readValue((InputStream) fs.open(fileStatus.getPath()), DataSegment.class); publishedSegmentsBuilder.add(segment); } return publishedSegmentsBuilder.build(); http://git-wip-us.apache.org/repos/asf/hive/blob/245c39b4/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java -- diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java index bb43d51..898f97a 100644 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java +++ b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java @@ -46,7 +46,8 @@ public class TestHiveDruidQueryBasedInputFormat extends TestCase { + "\"descending\":true," + "\"virtualColumns\":[]," + "\"filter\":null," - + "\"granularity\":\"DAY\"," + + "\"granularity\":{\"type\":\"period\",\"period\":\"P1D\",\"timeZone\":" + + "\"America/Los_Angeles\",\"origin\":null}," + "\"aggregations\":[]," + "\"postAggregations\":[]," + "\"context\":null}, [
[2/2] hive git commit: HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum)
HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9e10b88c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9e10b88c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9e10b88c Branch: refs/heads/branch-2.2 Commit: 9e10b88c33a3ae9cb09cb230c4ae09e442203ba9 Parents: a20e63e Author: Aihua Xu Authored: Thu Mar 8 11:33:37 2018 -0800 Committer: Aihua Xu Committed: Mon Mar 12 15:01:25 2018 -0700 -- .../TablePropertyEnrichmentOptimizer.java | 45 +++- .../avro_tableproperty_optimize.q | 63 ++ .../avro_tableproperty_optimize.q.out | 226 +++ 3 files changed, 324 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/9e10b88c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java index 5824490..154eb02 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -40,8 +41,10 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hive.common.util.ReflectionUtil; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -78,29 +81,51 @@ class TablePropertyEnrichmentOptimizer extends Transform { } } + /** + * Retrieves the table properties as well as the properties from Serde. + */ + private static Map getTableParameters(Table table) { +Map originalTableParameters = new HashMap<>(table.getParameters()); +Properties tableMetadata = MetaStoreUtils.getTableMetadata(table); +for (String property : tableMetadata.stringPropertyNames()) { + if (!originalTableParameters.containsKey(property)) { +originalTableParameters.put(property, tableMetadata.getProperty(property)); + } +} +return originalTableParameters; + } + private static class Processor implements NodeProcessor { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { TableScanOperator tsOp = (TableScanOperator) nd; WalkerCtx context = (WalkerCtx)procCtx; - TableScanDesc tableScanDesc = tsOp.getConf(); Table table = tsOp.getConf().getTableMetadata().getTTable(); - Map tableParameters = table.getParameters(); - Properties tableProperties = new Properties(); - tableProperties.putAll(tableParameters); - Deserializer deserializer = tableScanDesc.getTableMetadata().getDeserializer(); - String deserializerClassName = deserializer.getClass().getName(); + Map originalTableParameters = getTableParameters(table); + if (LOG.isDebugEnabled()) { +LOG.debug("Original Table parameters: " + originalTableParameters); + } + Properties clonedTableParameters = new Properties(); + clonedTableParameters.putAll(originalTableParameters); + + String deserializerClassName = null; try { +deserializerClassName = tableScanDesc.getTableMetadata().getSd().getSerdeInfo().getSerializationLib(); +Deserializer deserializer = ReflectionUtil.newInstance( +context.conf.getClassByName(deserializerClassName) +.asSubclass(Deserializer.class), +context.conf); + if (context.serdeClassesUnderConsideration.contains(deserializerClassName)) { - deserializer.initialize(context.conf, tableProperties); + deserializer.initialize(context.conf, clonedTableParameters); LOG.debug("SerDe init succeeded for class: " + deserializerClassName); - for (Map.Entry property : tableProperties.entrySet()) { -if (!property.getValue().equals(tableParameters.get(propert
[1/2] hive git commit: Revert "HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum)"
Repository: hive Updated Branches: refs/heads/branch-2.2 5949479f7 -> 9e10b88c3 Revert "HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum)" This reverts commit 5949479f7e08987b67c4ee86c06c2d5949f75bee. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a20e63e1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a20e63e1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a20e63e1 Branch: refs/heads/branch-2.2 Commit: a20e63e1eab675fcf2fa85ec9a320fec4fec6886 Parents: 5949479 Author: Aihua Xu Authored: Mon Mar 12 15:00:53 2018 -0700 Committer: Aihua Xu Committed: Mon Mar 12 15:00:53 2018 -0700 -- .../TablePropertyEnrichmentOptimizer.java | 45 +--- .../avro_tableproperty_optimize.q | 63 -- .../avro_tableproperty_optimize.q.out | 226 --- 3 files changed, 10 insertions(+), 324 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/a20e63e1/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java index d313f7d..5824490 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java @@ -26,7 +26,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -41,10 +40,8 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.serde2.Deserializer; -import org.apache.hive.common.util.ReflectionUtil; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -81,51 +78,29 @@ class TablePropertyEnrichmentOptimizer extends Transform { } } - /** - * Retrieves the table properties as well as the properties from Serde. - */ - private static Map getTableParameters(Table table) { -Map originalTableParameters = new HashMap<>(table.getParameters()); -Properties tableMetadata = MetaStoreUtils.getTableMetadata(table); -for (String property : tableMetadata.stringPropertyNames()) { - if (!originalTableParameters.containsKey(property)) { -originalTableParameters.put(property, tableMetadata.getProperty(property)); - } -} -return originalTableParameters; - } - private static class Processor implements NodeProcessor { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { TableScanOperator tsOp = (TableScanOperator) nd; WalkerCtx context = (WalkerCtx)procCtx; + TableScanDesc tableScanDesc = tsOp.getConf(); Table table = tsOp.getConf().getTableMetadata().getTTable(); + Map tableParameters = table.getParameters(); + Properties tableProperties = new Properties(); + tableProperties.putAll(tableParameters); - Map originalTableParameters = getTableParameters(table); - if (LOG.isDebugEnabled()) { -LOG.debug("Original Table parameters: " + originalTableParameters); - } - Properties clonedTableParameters = new Properties(); - clonedTableParameters.putAll(originalTableParameters); - - String deserializerClassName = null; + Deserializer deserializer = tableScanDesc.getTableMetadata().getDeserializer(); + String deserializerClassName = deserializer.getClass().getName(); try { -deserializerClassName = tableScanDesc.getTableMetadata().getSd().getSerdeInfo().getSerializationLib(); -Deserializer deserializer = ReflectionUtil.newInstance( -context.conf.getClassByName(deserializerClassName) -.asSubclass(Deserializer.class), -context.conf); - if (context.serdeClassesUnderConsideration.contains(deserializerClassName)) { - deserializer.initialize(context.conf, clonedTableParameters); + deserializer.initialize(context.conf, tableProperties); LOG.debug("SerDe init succeeded for class: " + d
[1/2] hive git commit: Revert "HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum)"
Repository: hive Updated Branches: refs/heads/branch-2.2 e8e3974d1 -> 5949479f7 Revert "HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum)" This reverts commit e8e3974d157a550ec0112258b59b603d9829. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/220d1998 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/220d1998 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/220d1998 Branch: refs/heads/branch-2.2 Commit: 220d199859e373f99e314e50121a149c10351a67 Parents: e8e3974 Author: Aihua Xu Authored: Mon Mar 12 14:52:36 2018 -0700 Committer: Aihua Xu Committed: Mon Mar 12 14:52:36 2018 -0700 -- .../TablePropertyEnrichmentOptimizer.java | 45 +--- .../avro_tableproperty_optimize.q | 63 -- .../avro_tableproperty_optimize.q.out | 226 --- 3 files changed, 10 insertions(+), 324 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/220d1998/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java index d313f7d..5824490 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java @@ -26,7 +26,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -41,10 +40,8 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.serde2.Deserializer; -import org.apache.hive.common.util.ReflectionUtil; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -81,51 +78,29 @@ class TablePropertyEnrichmentOptimizer extends Transform { } } - /** - * Retrieves the table properties as well as the properties from Serde. - */ - private static Map getTableParameters(Table table) { -Map originalTableParameters = new HashMap<>(table.getParameters()); -Properties tableMetadata = MetaStoreUtils.getTableMetadata(table); -for (String property : tableMetadata.stringPropertyNames()) { - if (!originalTableParameters.containsKey(property)) { -originalTableParameters.put(property, tableMetadata.getProperty(property)); - } -} -return originalTableParameters; - } - private static class Processor implements NodeProcessor { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { TableScanOperator tsOp = (TableScanOperator) nd; WalkerCtx context = (WalkerCtx)procCtx; + TableScanDesc tableScanDesc = tsOp.getConf(); Table table = tsOp.getConf().getTableMetadata().getTTable(); + Map tableParameters = table.getParameters(); + Properties tableProperties = new Properties(); + tableProperties.putAll(tableParameters); - Map originalTableParameters = getTableParameters(table); - if (LOG.isDebugEnabled()) { -LOG.debug("Original Table parameters: " + originalTableParameters); - } - Properties clonedTableParameters = new Properties(); - clonedTableParameters.putAll(originalTableParameters); - - String deserializerClassName = null; + Deserializer deserializer = tableScanDesc.getTableMetadata().getDeserializer(); + String deserializerClassName = deserializer.getClass().getName(); try { -deserializerClassName = tableScanDesc.getTableMetadata().getSd().getSerdeInfo().getSerializationLib(); -Deserializer deserializer = ReflectionUtil.newInstance( -context.conf.getClassByName(deserializerClassName) -.asSubclass(Deserializer.class), -context.conf); - if (context.serdeClassesUnderConsideration.contains(deserializerClassName)) { - deserializer.initialize(context.conf, clonedTableParameters); + deserializer.initialize(context.conf, tableProperties); LOG.debug("SerDe init succeeded for class: " + d
[2/2] hive git commit: HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum)
HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5949479f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5949479f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5949479f Branch: refs/heads/branch-2.2 Commit: 5949479f7e08987b67c4ee86c06c2d5949f75bee Parents: 220d199 Author: Aihua Xu Authored: Thu Mar 8 11:33:37 2018 -0800 Committer: Aihua Xu Committed: Mon Mar 12 14:52:53 2018 -0700 -- .../TablePropertyEnrichmentOptimizer.java | 45 +++- .../avro_tableproperty_optimize.q | 63 ++ .../avro_tableproperty_optimize.q.out | 226 +++ 3 files changed, 324 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/5949479f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java index 5824490..d313f7d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -40,8 +41,10 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hive.common.util.ReflectionUtil; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -78,29 +81,51 @@ class TablePropertyEnrichmentOptimizer extends Transform { } } + /** + * Retrieves the table properties as well as the properties from Serde. + */ + private static Map getTableParameters(Table table) { +Map originalTableParameters = new HashMap<>(table.getParameters()); +Properties tableMetadata = MetaStoreUtils.getTableMetadata(table); +for (String property : tableMetadata.stringPropertyNames()) { + if (!originalTableParameters.containsKey(property)) { +originalTableParameters.put(property, tableMetadata.getProperty(property)); + } +} +return originalTableParameters; + } + private static class Processor implements NodeProcessor { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { TableScanOperator tsOp = (TableScanOperator) nd; WalkerCtx context = (WalkerCtx)procCtx; - TableScanDesc tableScanDesc = tsOp.getConf(); Table table = tsOp.getConf().getTableMetadata().getTTable(); - Map tableParameters = table.getParameters(); - Properties tableProperties = new Properties(); - tableProperties.putAll(tableParameters); - Deserializer deserializer = tableScanDesc.getTableMetadata().getDeserializer(); - String deserializerClassName = deserializer.getClass().getName(); + Map originalTableParameters = getTableParameters(table); + if (LOG.isDebugEnabled()) { +LOG.debug("Original Table parameters: " + originalTableParameters); + } + Properties clonedTableParameters = new Properties(); + clonedTableParameters.putAll(originalTableParameters); + + String deserializerClassName = null; try { +deserializerClassName = tableScanDesc.getTableMetadata().getSd().getSerdeInfo().getSerializationLib(); +Deserializer deserializer = ReflectionUtil.newInstance( +context.conf.getClassByName(deserializerClassName) +.asSubclass(Deserializer.class), +context.conf); + if (context.serdeClassesUnderConsideration.contains(deserializerClassName)) { - deserializer.initialize(context.conf, tableProperties); + deserializer.initialize(context.conf, clonedTableParameters); LOG.debug("SerDe init succeeded for class: " + deserializerClassName); - for (Map.Entry property : tableProperties.entrySet()) { -if (!property.getValue().equals(tableParameters.get(p
hive git commit: HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum)
Repository: hive Updated Branches: refs/heads/branch-2 cbcd846b7 -> ad5a2fa0e HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ad5a2fa0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ad5a2fa0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ad5a2fa0 Branch: refs/heads/branch-2 Commit: ad5a2fa0eed89bf88a4287965274c4988e71ee98 Parents: cbcd846 Author: Aihua Xu Authored: Thu Mar 8 11:33:37 2018 -0800 Committer: Aihua Xu Committed: Mon Mar 12 14:51:43 2018 -0700 -- .../TablePropertyEnrichmentOptimizer.java | 45 +++- .../avro_tableproperty_optimize.q | 63 ++ .../avro_tableproperty_optimize.q.out | 226 +++ 3 files changed, 324 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/ad5a2fa0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java index 5824490..154eb02 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -40,8 +41,10 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hive.common.util.ReflectionUtil; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -78,29 +81,51 @@ class TablePropertyEnrichmentOptimizer extends Transform { } } + /** + * Retrieves the table properties as well as the properties from Serde. + */ + private static Map getTableParameters(Table table) { +Map originalTableParameters = new HashMap<>(table.getParameters()); +Properties tableMetadata = MetaStoreUtils.getTableMetadata(table); +for (String property : tableMetadata.stringPropertyNames()) { + if (!originalTableParameters.containsKey(property)) { +originalTableParameters.put(property, tableMetadata.getProperty(property)); + } +} +return originalTableParameters; + } + private static class Processor implements NodeProcessor { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { TableScanOperator tsOp = (TableScanOperator) nd; WalkerCtx context = (WalkerCtx)procCtx; - TableScanDesc tableScanDesc = tsOp.getConf(); Table table = tsOp.getConf().getTableMetadata().getTTable(); - Map tableParameters = table.getParameters(); - Properties tableProperties = new Properties(); - tableProperties.putAll(tableParameters); - Deserializer deserializer = tableScanDesc.getTableMetadata().getDeserializer(); - String deserializerClassName = deserializer.getClass().getName(); + Map originalTableParameters = getTableParameters(table); + if (LOG.isDebugEnabled()) { +LOG.debug("Original Table parameters: " + originalTableParameters); + } + Properties clonedTableParameters = new Properties(); + clonedTableParameters.putAll(originalTableParameters); + + String deserializerClassName = null; try { +deserializerClassName = tableScanDesc.getTableMetadata().getSd().getSerdeInfo().getSerializationLib(); +Deserializer deserializer = ReflectionUtil.newInstance( +context.conf.getClassByName(deserializerClassName) +.asSubclass(Deserializer.class), +context.conf); + if (context.serdeClassesUnderConsideration.contains(deserializerClassName)) { - deserializer.initialize(context.conf, tableProperties); + deserializer.initialize(context.conf, clonedTableParameters); LOG.debug("SerDe init succeeded for class: " + deserializerClassName); - for (Map.Entry property : tableProperties.entrySe
hive git commit: HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum)
Repository: hive Updated Branches: refs/heads/branch-2.2 5fee15988 -> e8e3974d1 HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e8e3974d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e8e3974d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e8e3974d Branch: refs/heads/branch-2.2 Commit: e8e3974d157a550ec0112258b59b603d9829 Parents: 5fee159 Author: Aihua Xu Authored: Thu Mar 8 11:33:37 2018 -0800 Committer: Aihua Xu Committed: Mon Mar 12 14:39:50 2018 -0700 -- .../TablePropertyEnrichmentOptimizer.java | 45 +++- .../avro_tableproperty_optimize.q | 63 ++ .../avro_tableproperty_optimize.q.out | 226 +++ 3 files changed, 324 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/e8e3974d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java index 5824490..d313f7d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -40,8 +41,10 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hive.common.util.ReflectionUtil; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -78,29 +81,51 @@ class TablePropertyEnrichmentOptimizer extends Transform { } } + /** + * Retrieves the table properties as well as the properties from Serde. + */ + private static Map getTableParameters(Table table) { +Map originalTableParameters = new HashMap<>(table.getParameters()); +Properties tableMetadata = MetaStoreUtils.getTableMetadata(table); +for (String property : tableMetadata.stringPropertyNames()) { + if (!originalTableParameters.containsKey(property)) { +originalTableParameters.put(property, tableMetadata.getProperty(property)); + } +} +return originalTableParameters; + } + private static class Processor implements NodeProcessor { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { TableScanOperator tsOp = (TableScanOperator) nd; WalkerCtx context = (WalkerCtx)procCtx; - TableScanDesc tableScanDesc = tsOp.getConf(); Table table = tsOp.getConf().getTableMetadata().getTTable(); - Map tableParameters = table.getParameters(); - Properties tableProperties = new Properties(); - tableProperties.putAll(tableParameters); - Deserializer deserializer = tableScanDesc.getTableMetadata().getDeserializer(); - String deserializerClassName = deserializer.getClass().getName(); + Map originalTableParameters = getTableParameters(table); + if (LOG.isDebugEnabled()) { +LOG.debug("Original Table parameters: " + originalTableParameters); + } + Properties clonedTableParameters = new Properties(); + clonedTableParameters.putAll(originalTableParameters); + + String deserializerClassName = null; try { +deserializerClassName = tableScanDesc.getTableMetadata().getSd().getSerdeInfo().getSerializationLib(); +Deserializer deserializer = ReflectionUtil.newInstance( +context.conf.getClassByName(deserializerClassName) +.asSubclass(Deserializer.class), +context.conf); + if (context.serdeClassesUnderConsideration.contains(deserializerClassName)) { - deserializer.initialize(context.conf, tableProperties); + deserializer.initialize(context.conf, clonedTableParameters); LOG.debug("SerDe init succeeded for class: " + deserializerClassName); - for (Map.Entry property : tableProperti
hive git commit: HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum)
Repository: hive Updated Branches: refs/heads/master 190c72e77 -> 1629ec058 HIVE-14792: AvroSerde reads the remote schema-file at least once per mapper, per table reference. (Addendum) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1629ec05 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1629ec05 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1629ec05 Branch: refs/heads/master Commit: 1629ec058faf2dce581de4f393f0c6485c7425d7 Parents: 190c72e Author: Aihua Xu Authored: Thu Mar 8 11:33:37 2018 -0800 Committer: Aihua Xu Committed: Mon Mar 12 14:16:38 2018 -0700 -- .../TablePropertyEnrichmentOptimizer.java | 45 +++- .../avro_tableproperty_optimize.q | 63 ++ .../avro_tableproperty_optimize.q.out | 226 +++ 3 files changed, 324 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/1629ec05/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java index d806775..bc17691 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TablePropertyEnrichmentOptimizer.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -40,8 +41,10 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hive.common.util.ReflectionUtil; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -78,29 +81,51 @@ class TablePropertyEnrichmentOptimizer extends Transform { } } + /** + * Retrieves the table properties as well as the properties from Serde. + */ + private static Map getTableParameters(Table table) { +Map originalTableParameters = new HashMap<>(table.getParameters()); +Properties tableMetadata = MetaStoreUtils.getTableMetadata(table); +for (String property : tableMetadata.stringPropertyNames()) { + if (!originalTableParameters.containsKey(property)) { +originalTableParameters.put(property, tableMetadata.getProperty(property)); + } +} +return originalTableParameters; + } + private static class Processor implements NodeProcessor { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { TableScanOperator tsOp = (TableScanOperator) nd; WalkerCtx context = (WalkerCtx)procCtx; - TableScanDesc tableScanDesc = tsOp.getConf(); Table table = tsOp.getConf().getTableMetadata().getTTable(); - Map tableParameters = table.getParameters(); - Properties tableProperties = new Properties(); - tableProperties.putAll(tableParameters); - Deserializer deserializer = tableScanDesc.getTableMetadata().getDeserializer(); - String deserializerClassName = deserializer.getClass().getName(); + Map originalTableParameters = getTableParameters(table); + if (LOG.isDebugEnabled()) { +LOG.debug("Original Table parameters: " + originalTableParameters); + } + Properties clonedTableParameters = new Properties(); + clonedTableParameters.putAll(originalTableParameters); + + String deserializerClassName = null; try { +deserializerClassName = tableScanDesc.getTableMetadata().getSd().getSerdeInfo().getSerializationLib(); +Deserializer deserializer = ReflectionUtil.newInstance( +context.conf.getClassByName(deserializerClassName) +.asSubclass(Deserializer.class), +context.conf); + if (context.serdeClassesUnderConsideration.contains(deserializerClassName)) { - deserializer.initialize(context.conf, tableProperties); + deserializer.initialize(context.conf, clonedTableParameters); LOG.debug("SerDe init succeeded for class: " + deserializerClassName); - for (Map.Entry property : tableProperties.entry
[1/2] hive git commit: HIVE-17735: ObjectStore.addNotificationEvent is leaking queries (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master 8cf36e733 -> 35605732b HIVE-17735: ObjectStore.addNotificationEvent is leaking queries (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ec7ccc3a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ec7ccc3a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ec7ccc3a Branch: refs/heads/master Commit: ec7ccc3a452fa125719ca820b5f751ddd00686ec Parents: 8cf36e7 Author: Aihua Xu Authored: Mon Feb 5 15:35:30 2018 -0800 Committer: Aihua Xu Committed: Tue Feb 13 13:04:15 2018 -0800 -- .../hadoop/hive/metastore/ObjectStore.java | 42 +++- .../hadoop/hive/metastore/TestObjectStore.java | 2 +- 2 files changed, 15 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/ec7ccc3a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java -- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index d58ed67..edabaa1 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -3941,13 +3941,13 @@ public class ObjectStore implements RawStore, Configurable { } boolean success = false; -QueryWrapper queryWrapper = new QueryWrapper(); +Query query = null; try { openTransaction(); LOG.debug("execute removeUnusedColumnDescriptor"); - Query query = pm.newQuery("select count(1) from " + + query = pm.newQuery("select count(1) from " + "org.apache.hadoop.hive.metastore.model.MStorageDescriptor where (this.cd == inCD)"); query.declareParameters("MColumnDescriptor inCD"); long count = ((Long)query.execute(oldCD)).longValue(); @@ -3960,7 +3960,7 @@ public class ObjectStore implements RawStore, Configurable { success = commitTransaction(); LOG.debug("successfully deleted a CD in removeUnusedColumnDescriptor"); } finally { - rollbackAndCleanup(success, queryWrapper); + rollbackAndCleanup(success, query); } } @@ -8819,14 +8819,13 @@ public class ObjectStore implements RawStore, Configurable { public Function getFunction(String dbName, String funcName) throws MetaException { boolean commited = false; Function func = null; +Query query = null; try { openTransaction(); func = convertToFunction(getMFunction(dbName, funcName)); commited = commitTransaction(); } finally { - if (!commited) { -rollbackTransaction(); - } + rollbackAndCleanup(commited, query); } return func; } @@ -8834,17 +8833,16 @@ public class ObjectStore implements RawStore, Configurable { @Override public List getAllFunctions() throws MetaException { boolean commited = false; +Query query = null; try { openTransaction(); - Query query = pm.newQuery(MFunction.class); + query = pm.newQuery(MFunction.class); List allFunctions = (List) query.execute(); pm.retrieveAll(allFunctions); commited = commitTransaction(); return convertToFunctions(allFunctions); } finally { - if (!commited) { -rollbackTransaction(); - } + rollbackAndCleanup(commited, query); } } @@ -8905,10 +8903,7 @@ public class ObjectStore implements RawStore, Configurable { } return result; } finally { - if (!commited) { -rollbackAndCleanup(commited, query); -return null; - } + rollbackAndCleanup(commited, query); } } @@ -8938,6 +8933,7 @@ public class ObjectStore implements RawStore, Configurable { query.setUnique(true); // only need to execute it to get db Lock query.execute(); + query.closeAll(); }).run(); } @@ -9003,8 +8999,8 @@ public class ObjectStore implements RawStore, Configurable { try { openTransaction(); lockForUpdate(); - Query objectQuery = pm.newQuery(MNotificationNextId.class); - Collection ids = (Collection) objectQuery.execute(); + query = pm.newQuery(MNotificationNextId.class); + Collection ids = (Collection) query.execute(); MNotificationNextId mNotificationNextId = null; boolean needToPersistId; if (CollectionUtils.isEmpty(ids)) { @@ -9533,12 +9529,7 @@ public class ObjectStore implements RawStore, Configurable { } commited = commitTransaction(); } finally { - if (!commited) { -rollbackTransac
[2/2] hive git commit: HIVE-18586: Upgrade Derby to 10.14.1.0 (Janaki Lahorani, reviewed by Aihua Xu)
HIVE-18586: Upgrade Derby to 10.14.1.0 (Janaki Lahorani, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/35605732 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/35605732 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/35605732 Branch: refs/heads/master Commit: 35605732b2041eee809485718bfd951cdfae0980 Parents: ec7ccc3 Author: Aihua Xu Authored: Tue Feb 13 13:06:31 2018 -0800 Committer: Aihua Xu Committed: Tue Feb 13 13:06:31 2018 -0800 -- .../org/apache/hive/hcatalog/DerbyPolicy.java | 90 .../org/apache/hive/hcatalog/DerbyPolicy.java | 90 .../apache/hive/hcatalog/cli/TestPermsGrp.java | 3 + .../mapreduce/TestHCatPartitionPublish.java | 3 + .../org/apache/hive/hcatalog/package-info.java | 22 + .../hive/hcatalog/api/TestHCatClient.java | 4 + pom.xml | 2 +- .../metastore/TestHiveMetaStoreGetMetaConf.java | 25 -- .../TestHiveMetaStorePartitionSpecs.java| 26 -- 9 files changed, 213 insertions(+), 52 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/35605732/core/src/test/java/org/apache/hive/hcatalog/DerbyPolicy.java -- diff --git a/core/src/test/java/org/apache/hive/hcatalog/DerbyPolicy.java b/core/src/test/java/org/apache/hive/hcatalog/DerbyPolicy.java new file mode 100644 index 000..cecf6dc --- /dev/null +++ b/core/src/test/java/org/apache/hive/hcatalog/DerbyPolicy.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.hcatalog; + +import org.apache.derby.security.SystemPermission; + +import java.security.CodeSource; +import java.security.Permission; +import java.security.PermissionCollection; +import java.security.Policy; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.Iterator; + +/** + * A security policy that grants usederbyinternals + * + * + * HCatalog tests use Security Manager to handle exits. With Derby version 10.14.1, if a + * security manager is configured, embedded Derby requires usederbyinternals permission, and + * that is checked directly using AccessController.checkPermission. This class will be used to + * setup a security policy to grant usederbyinternals, in tests that use NoExitSecurityManager. + * + */ +public class DerbyPolicy extends Policy { + + private static PermissionCollection perms; + + public DerbyPolicy() { +super(); +if (perms == null) { + perms = new DerbyPermissionCollection(); + addPermissions(); +} + } + + @Override + public PermissionCollection getPermissions(CodeSource codesource) { +return perms; + } + + private void addPermissions() { +SystemPermission systemPermission = new SystemPermission("engine", "usederbyinternals"); +perms.add(systemPermission); + } + + class DerbyPermissionCollection extends PermissionCollection { + +ArrayList perms = new ArrayList(); + +public void add(Permission p) { + perms.add(p); +} + +public boolean implies(Permission p) { + for (Iterator i = perms.iterator(); i.hasNext();) { +if (((Permission) i.next()).implies(p)) { + return true; +} + } + return false; +} + +public Enumeration elements() { + return Collections.enumeration(perms); +} + +public boolean isReadOnly() { + return false; +} + } +} + http://git-wip-us.apache.org/repos/asf/hive/blob/35605732/hcatalog/core/src/test/java/org/apache/hive/hcatalog/DerbyPolicy.java -- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/DerbyPolicy.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/DerbyPolicy.java new file mode 100644 index 000..cecf6dc --- /dev/null +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/DerbyPo
hive git commit: HIVE-18550: Keep the hbase table name property as hbase.table.name (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master fa14a4365 -> 1eddbc06a HIVE-18550: Keep the hbase table name property as hbase.table.name (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1eddbc06 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1eddbc06 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1eddbc06 Branch: refs/heads/master Commit: 1eddbc06a6224cb860ecb2f331cb2462a57b228b Parents: fa14a43 Author: Aihua Xu Authored: Fri Jan 26 15:30:52 2018 -0800 Committer: Aihua Xu Committed: Mon Feb 12 11:03:25 2018 -0800 -- .../hadoop/hive/hbase/HiveHFileOutputFormat.java | 14 +- hbase-handler/src/test/queries/positive/hbase_bulk.q | 2 +- .../src/test/queries/positive/hbase_handler_bulk.q| 4 ++-- .../test/results/positive/hbase_handler_bulk.q.out| 8 4 files changed, 20 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/1eddbc06/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java -- diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java index d8dad06..4fa0272 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.shims.ShimLoader; @@ -64,7 +65,8 @@ public class HiveHFileOutputFormat extends HiveOutputFormat { public static final String HFILE_FAMILY_PATH = "hfile.family.path"; - + public static final String OUTPUT_TABLE_NAME_CONF_KEY = + "hbase.mapreduce.hfileoutputformat.table.name"; static final Logger LOG = LoggerFactory.getLogger(HiveHFileOutputFormat.class.getName()); private @@ -95,6 +97,16 @@ public class HiveHFileOutputFormat extends Properties tableProperties, final Progressable progressable) throws IOException { +String hbaseTableName = jc.get(HBaseSerDe.HBASE_TABLE_NAME); +if (hbaseTableName == null) { + hbaseTableName = tableProperties.getProperty(hive_metastoreConstants.META_TABLE_NAME); + hbaseTableName = hbaseTableName.toLowerCase(); + if (hbaseTableName.startsWith(HBaseStorageHandler.DEFAULT_PREFIX)) { +hbaseTableName = hbaseTableName.substring(HBaseStorageHandler.DEFAULT_PREFIX.length()); + } +} +jc.set(OUTPUT_TABLE_NAME_CONF_KEY, hbaseTableName); + // Read configuration for the target path, first from jobconf, then from table properties String hfilePath = getFamilyPath(jc, tableProperties); if (hfilePath == null) { http://git-wip-us.apache.org/repos/asf/hive/blob/1eddbc06/hbase-handler/src/test/queries/positive/hbase_bulk.q -- diff --git a/hbase-handler/src/test/queries/positive/hbase_bulk.q b/hbase-handler/src/test/queries/positive/hbase_bulk.q index 5e0c14e..475aafc 100644 --- a/hbase-handler/src/test/queries/positive/hbase_bulk.q +++ b/hbase-handler/src/test/queries/positive/hbase_bulk.q @@ -9,7 +9,7 @@ create table hbsort(key string, val string, val2 string) stored as INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.hbase.HiveHFileOutputFormat' -TBLPROPERTIES ('hfile.family.path' = '/tmp/hbsort/cf','hbase.mapreduce.hfileoutputformat.table.name'='hbsort'); +TBLPROPERTIES ('hfile.family.path' = '/tmp/hbsort/cf'); -- this is a dummy table used for controlling how the input file -- for TotalOrderPartitioner is created http://git-wip-us.apache.org/repos/asf/hive/blob/1eddbc06/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q -- diff --git a/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q b/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q index 5ac4dc8..d02a61f 100644 --- a/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q +++ b/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q @@ -6,7 +6,7 @@ drop table if exists hb_target; create table hb_target(key int, val string) stored by 'org.apache.hadoop.hive.hbase.HBaseStor
hive git commit: HIVE-18426: Memory leak in RoutingAppender for every hive operation (kalyan kumar kalvagadda, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master d2a14fe28 -> 90f26bb44 HIVE-18426: Memory leak in RoutingAppender for every hive operation (kalyan kumar kalvagadda, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/90f26bb4 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/90f26bb4 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/90f26bb4 Branch: refs/heads/master Commit: 90f26bb443176df01a402b80d344b16abfb33eff Parents: d2a14fe Author: Aihua Xu Authored: Mon Jan 29 15:21:23 2018 -0800 Committer: Aihua Xu Committed: Mon Jan 29 15:21:23 2018 -0800 -- .../org/apache/hadoop/hive/common/LogUtils.java | 11 +- .../operation/TestOperationLoggingLayout.java | 109 +++ 2 files changed, 88 insertions(+), 32 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/90f26bb4/common/src/java/org/apache/hadoop/hive/common/LogUtils.java -- diff --git a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java index 5c7ec69..5068eb5 100644 --- a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java @@ -29,10 +29,8 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.Appender; -import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.routing.RoutingAppender; -import org.apache.logging.log4j.core.config.AppenderControl; import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.core.config.LoggerConfig; import org.apache.logging.log4j.core.impl.Log4jContextFactory; @@ -248,14 +246,9 @@ public class LogUtils { // The appender is configured to use ${ctx:queryId} by registerRoutingAppender() try { Class clazz = routingAppender.getClass(); -Method method = clazz.getDeclaredMethod("getControl", String.class, LogEvent.class); +Method method = clazz.getDeclaredMethod("deleteAppender", String.class); method.setAccessible(true); -AppenderControl control = (AppenderControl) method.invoke(routingAppender, queryId, null); -Appender subordinateAppender = control.getAppender(); -if (!subordinateAppender.isStopped()) { - // this will cause the subordinate appender to close its output stream. - subordinateAppender.stop(); -} +method.invoke(routingAppender, queryId); } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { l4j.warn("Unable to close the operation log appender for query id " + queryId, e); http://git-wip-us.apache.org/repos/asf/hive/blob/90f26bb4/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java index 8febe3e..d90d590 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java @@ -102,14 +102,8 @@ public class TestOperationLoggingLayout { miniHS2.stop(); } - @Test - public void testSwitchLogLayout() throws Exception { -// verify whether the sql operation log is generated and fetch correctly. -OperationHandle operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null); -RowSet rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 1000, -FetchType.LOG); + private String getQueryId(RowSet rowSetLog) { Iterator iter = rowSetLog.iterator(); -String queryId = null; // non-verbose pattern is %-5p : %m%n. Look for " : " while (iter.hasNext()) { String row = iter.next()[0].toString(); @@ -119,9 +113,32 @@ public class TestOperationLoggingLayout { String queryIdLoggingProbe = "INFO : Query ID = "; int index = row.indexOf(queryIdLoggingProbe); if (index >= 0) { -queryId = row.substring(queryIdLoggingProbe.length()).trim(); +return row.substring(queryIdLoggingProbe.length()).trim(); } } +return null; + } + + private void appendHushableRandomAc
[2/3] hive git commit: Revert "HIVE-18202: Automatically migrate hbase.table.name to hbase.mapreduce.hfileoutputformat.table.name for hbase-based table (addendum)"
Revert "HIVE-18202: Automatically migrate hbase.table.name to hbase.mapreduce.hfileoutputformat.table.name for hbase-based table (addendum)" This reverts commit ee802dba3f28d0467bbea045e6aa5c9bfac8e2a5. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c14a46dc Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c14a46dc Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c14a46dc Branch: refs/heads/master Commit: c14a46dc48ccc03ff135588c20571a522c2cb2b9 Parents: 0fc1cd4 Author: Aihua Xu Authored: Fri Jan 26 13:44:05 2018 -0800 Committer: Aihua Xu Committed: Fri Jan 26 13:44:05 2018 -0800 -- metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/c14a46dc/metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql -- diff --git a/metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql b/metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql index c054322..0fdc615 100644 --- a/metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql +++ b/metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql @@ -1,6 +1,6 @@ -UPDATE "TABLE_PARAMS" - SET "PARAM_KEY" = 'hbase.mapreduce.hfileoutputformat.table.name' +UPDATE TABLE_PARAMS + SET PARAM_KEY = 'hbase.mapreduce.hfileoutputformat.table.name' WHERE - "PARAM_KEY" = 'hbase.table.name' + PARAM_KEY = 'hbase.table.name' ;
[1/3] hive git commit: Revert "HIVE-18366: Update HBaseSerDe to use hbase.mapreduce.hfileoutputformat.table.name instead of hbase.table.name as the table name property (Aihua Xu, reviewed Yongzhi Chen
Repository: hive Updated Branches: refs/heads/master 255cf4ab2 -> 6b349b5f8 Revert "HIVE-18366: Update HBaseSerDe to use hbase.mapreduce.hfileoutputformat.table.name instead of hbase.table.name as the table name property (Aihua Xu, reviewed Yongzhi Chen)" This reverts commit 146234906982c59953d2dad25c0f1e8978126243. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0fc1cd4a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0fc1cd4a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0fc1cd4a Branch: refs/heads/master Commit: 0fc1cd4a33e4de9060b50da9ff8b54a74f00579b Parents: 255cf4a Author: Aihua Xu Authored: Fri Jan 26 13:43:38 2018 -0800 Committer: Aihua Xu Committed: Fri Jan 26 13:43:38 2018 -0800 -- .../apache/hadoop/hive/hbase/HBaseSerDe.java| 2 +- .../src/test/queries/negative/cascade_dbdrop.q | 2 +- .../queries/negative/cascade_dbdrop_hadoop20.q | 2 +- .../src/test/queries/negative/hbase_ddl.q | 2 +- .../test/queries/positive/external_table_ppd.q | 2 +- .../hbase_binary_external_table_queries.q | 6 +-- .../queries/positive/hbase_binary_map_queries.q | 16 +++ .../positive/hbase_binary_map_queries_prefix.q | 6 +-- .../positive/hbase_binary_storage_queries.q | 10 ++-- .../test/queries/positive/hbase_custom_key.q| 4 +- .../test/queries/positive/hbase_custom_key2.q | 2 +- .../test/queries/positive/hbase_custom_key3.q | 2 +- .../src/test/queries/positive/hbase_ddl.q | 2 +- .../src/test/queries/positive/hbase_queries.q | 6 +-- .../queries/positive/hbase_tablename_property.q | 14 -- .../queries/positive/hbase_timestamp_format.q | 4 +- .../src/test/queries/positive/hbase_viewjoins.q | 4 +- .../test/results/negative/cascade_dbdrop.q.out | 4 +- .../negative/cascade_dbdrop_hadoop20.q.out | 4 +- .../src/test/results/negative/hbase_ddl.q.out | 4 +- .../results/positive/external_table_ppd.q.out | 6 +-- .../hbase_binary_external_table_queries.q.out | 12 ++--- .../positive/hbase_binary_map_queries.q.out | 32 ++--- .../hbase_binary_map_queries_prefix.q.out | 12 ++--- .../positive/hbase_binary_storage_queries.q.out | 24 +- .../results/positive/hbase_custom_key.q.out | 8 ++-- .../results/positive/hbase_custom_key2.q.out| 4 +- .../results/positive/hbase_custom_key3.q.out| 4 +- .../src/test/results/positive/hbase_ddl.q.out | 8 ++-- .../test/results/positive/hbase_queries.q.out | 14 +++--- .../positive/hbase_tablename_property.q.out | 49 .../positive/hbase_timestamp_format.q.out | 8 ++-- .../test/results/positive/hbase_viewjoins.q.out | 8 ++-- .../hive/hcatalog/templeton/TestDesc.java | 2 +- .../hbase/TestPigHBaseStorageHandler.java | 21 ++--- .../hadoop/hive/hbase/HBaseQTestUtil.java | 2 +- 36 files changed, 128 insertions(+), 184 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/0fc1cd4a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java -- diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java index f203b8f..1553525 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java @@ -71,7 +71,7 @@ public class HBaseSerDe extends AbstractSerDe { public static final Logger LOG = LoggerFactory.getLogger(HBaseSerDe.class); public static final String HBASE_COLUMNS_MAPPING = "hbase.columns.mapping"; - public static final String HBASE_TABLE_NAME = "hbase.mapreduce.hfileoutputformat.table.name"; + public static final String HBASE_TABLE_NAME = "hbase.table.name"; public static final String HBASE_TABLE_DEFAULT_STORAGE_TYPE = "hbase.table.default.storage.type"; public static final String HBASE_KEY_COL = ":key"; public static final String HBASE_TIMESTAMP_COL = ":timestamp"; http://git-wip-us.apache.org/repos/asf/hive/blob/0fc1cd4a/hbase-handler/src/test/queries/negative/cascade_dbdrop.q -- diff --git a/hbase-handler/src/test/queries/negative/cascade_dbdrop.q b/hbase-handler/src/test/queries/negative/cascade_dbdrop.q index 39ecc6b..7f9df5e 100644 --- a/hbase-handler/src/test/queries/negative/cascade_dbdrop.q +++ b/hbase-handler/src/test/queries/negative/cascade_dbdrop.q @@ -14,7 +14,7 @@ CREATE DATABASE hbaseDB; CREATE TABLE hbaseDB.hbase_table_0(key int, value string) STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string") -TBLPROPERTIES ("hbase.mapr
[3/3] hive git commit: Revert "HIVE-18202: Automatically migrate hbase.table.name to hbase.mapreduce.hfileoutputformat.table.name for hbase-based table (Aihua Xu, reviewed by Naveen Gangam)"
Revert "HIVE-18202: Automatically migrate hbase.table.name to hbase.mapreduce.hfileoutputformat.table.name for hbase-based table (Aihua Xu, reviewed by Naveen Gangam)" This reverts commit d37b8026ed25b4150fd3782331e6800459d40d7a. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6b349b5f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6b349b5f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6b349b5f Branch: refs/heads/master Commit: 6b349b5f83cff54c4b6b04537227087b2b3d4a0d Parents: c14a46d Author: Aihua Xu Authored: Fri Jan 26 13:45:51 2018 -0800 Committer: Aihua Xu Committed: Fri Jan 26 13:45:51 2018 -0800 -- metastore/scripts/upgrade/derby/047-HIVE-18202.derby.sql | 6 -- .../scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 1 - metastore/scripts/upgrade/mssql/032-HIVE-18202.mssql.sql | 6 -- .../scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 1 - metastore/scripts/upgrade/mysql/047-HIVE-18202.mysql.sql | 6 -- .../scripts/upgrade/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 1 - metastore/scripts/upgrade/oracle/047-HIVE-18202-oracle.sql | 6 -- .../scripts/upgrade/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 1 - metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql | 6 -- .../upgrade/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql | 1 - 10 files changed, 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/6b349b5f/metastore/scripts/upgrade/derby/047-HIVE-18202.derby.sql -- diff --git a/metastore/scripts/upgrade/derby/047-HIVE-18202.derby.sql b/metastore/scripts/upgrade/derby/047-HIVE-18202.derby.sql deleted file mode 100644 index 0fdc615..000 --- a/metastore/scripts/upgrade/derby/047-HIVE-18202.derby.sql +++ /dev/null @@ -1,6 +0,0 @@ -UPDATE TABLE_PARAMS - SET PARAM_KEY = 'hbase.mapreduce.hfileoutputformat.table.name' -WHERE - PARAM_KEY = 'hbase.table.name' -; - http://git-wip-us.apache.org/repos/asf/hive/blob/6b349b5f/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql -- diff --git a/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql index 4372498..3a11881 100644 --- a/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -5,7 +5,6 @@ RUN '043-HIVE-16922.derby.sql'; RUN '044-HIVE-16997.derby.sql'; RUN '045-HIVE-16886.derby.sql'; RUN '046-HIVE-17566.derby.sql'; -RUN '047-HIVE-18202.derby.sql'; RUN '048-HIVE-14498.derby.sql'; RUN '049-HIVE-18489.derby.sql'; http://git-wip-us.apache.org/repos/asf/hive/blob/6b349b5f/metastore/scripts/upgrade/mssql/032-HIVE-18202.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/032-HIVE-18202.mssql.sql b/metastore/scripts/upgrade/mssql/032-HIVE-18202.mssql.sql deleted file mode 100644 index 0fdc615..000 --- a/metastore/scripts/upgrade/mssql/032-HIVE-18202.mssql.sql +++ /dev/null @@ -1,6 +0,0 @@ -UPDATE TABLE_PARAMS - SET PARAM_KEY = 'hbase.mapreduce.hfileoutputformat.table.name' -WHERE - PARAM_KEY = 'hbase.table.name' -; - http://git-wip-us.apache.org/repos/asf/hive/blob/6b349b5f/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index 52727f3..f5a260e 100644 --- a/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -6,7 +6,6 @@ SELECT 'Upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; :r 029-HIVE-16997.mssql.sql :r 030-HIVE-16886.mssql.sql :r 031-HIVE-17566.mssql.sql -:r 032-HIVE-18202.mssql.sql :r 033-HIVE-14498.mssql.sql :r 034-HIVE-18489.mssql.sql http://git-wip-us.apache.org/repos/asf/hive/blob/6b349b5f/metastore/scripts/upgrade/mysql/047-HIVE-18202.mysql.sql -- diff --git a/metastore/scripts/upgrade/mysql/047-HIVE-18202.mysql.sql b/metastore/scripts/upgrade/mysql/047-HIVE-18202.mysql.sql deleted file mode 100644 index 0fdc615..000 --- a/metastore/scripts/upgrade/mysql/047-HIVE-18202.mysql.sql +++ /dev/null @@ -1,6 +0,0 @@ -UPDATE TABLE_PARAMS - SET PARAM_KEY = 'hbase.mapreduce.hfileoutputformat.table.name' -WHERE - PARAM_KEY = 'hbase.table.name' -; - http://git-wip-us.apache.org/repos/asf/hive/blob/6b349b5f/metastore/sc
hive git commit: HIVE-18202: Automatically migrate hbase.table.name to hbase.mapreduce.hfileoutputformat.table.name for hbase-based table (addendum)
Repository: hive Updated Branches: refs/heads/master 042698ce2 -> ee802dba3 HIVE-18202: Automatically migrate hbase.table.name to hbase.mapreduce.hfileoutputformat.table.name for hbase-based table (addendum) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ee802dba Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ee802dba Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ee802dba Branch: refs/heads/master Commit: ee802dba3f28d0467bbea045e6aa5c9bfac8e2a5 Parents: 042698c Author: Aihua Xu Authored: Wed Jan 24 13:32:02 2018 -0800 Committer: Aihua Xu Committed: Wed Jan 24 13:32:02 2018 -0800 -- metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/ee802dba/metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql -- diff --git a/metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql b/metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql index 0fdc615..c054322 100644 --- a/metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql +++ b/metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql @@ -1,6 +1,6 @@ -UPDATE TABLE_PARAMS - SET PARAM_KEY = 'hbase.mapreduce.hfileoutputformat.table.name' +UPDATE "TABLE_PARAMS" + SET "PARAM_KEY" = 'hbase.mapreduce.hfileoutputformat.table.name' WHERE - PARAM_KEY = 'hbase.table.name' + "PARAM_KEY" = 'hbase.table.name' ;
hive git commit: HIVE-18202: Automatically migrate hbase.table.name to hbase.mapreduce.hfileoutputformat.table.name for hbase-based table (Aihua Xu, reviewed by Naveen Gangam)
Repository: hive Updated Branches: refs/heads/master 146234906 -> d37b8026e HIVE-18202: Automatically migrate hbase.table.name to hbase.mapreduce.hfileoutputformat.table.name for hbase-based table (Aihua Xu, reviewed by Naveen Gangam) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d37b8026 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d37b8026 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d37b8026 Branch: refs/heads/master Commit: d37b8026ed25b4150fd3782331e6800459d40d7a Parents: 1462349 Author: Aihua Xu Authored: Tue Jan 9 15:46:04 2018 -0800 Committer: Aihua Xu Committed: Tue Jan 9 15:46:04 2018 -0800 -- metastore/scripts/upgrade/derby/047-HIVE-18202.derby.sql | 6 ++ .../scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 1 + metastore/scripts/upgrade/mssql/032-HIVE-18202.mssql.sql | 6 ++ .../scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 1 + metastore/scripts/upgrade/mysql/047-HIVE-18202.mysql.sql | 6 ++ .../scripts/upgrade/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 1 + metastore/scripts/upgrade/oracle/047-HIVE-18202-oracle.sql | 6 ++ .../scripts/upgrade/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 1 + metastore/scripts/upgrade/postgres/046-HIVE-18202.postgres.sql | 6 ++ .../upgrade/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql | 1 + 10 files changed, 35 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/d37b8026/metastore/scripts/upgrade/derby/047-HIVE-18202.derby.sql -- diff --git a/metastore/scripts/upgrade/derby/047-HIVE-18202.derby.sql b/metastore/scripts/upgrade/derby/047-HIVE-18202.derby.sql new file mode 100644 index 000..0fdc615 --- /dev/null +++ b/metastore/scripts/upgrade/derby/047-HIVE-18202.derby.sql @@ -0,0 +1,6 @@ +UPDATE TABLE_PARAMS + SET PARAM_KEY = 'hbase.mapreduce.hfileoutputformat.table.name' +WHERE + PARAM_KEY = 'hbase.table.name' +; + http://git-wip-us.apache.org/repos/asf/hive/blob/d37b8026/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql -- diff --git a/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql index 1f2647d..75b0e73 100644 --- a/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -5,5 +5,6 @@ RUN '043-HIVE-16922.derby.sql'; RUN '044-HIVE-16997.derby.sql'; RUN '045-HIVE-16886.derby.sql'; RUN '046-HIVE-17566.derby.sql'; +RUN '047-HIVE-18202.derby.sql'; UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/d37b8026/metastore/scripts/upgrade/mssql/032-HIVE-18202.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/032-HIVE-18202.mssql.sql b/metastore/scripts/upgrade/mssql/032-HIVE-18202.mssql.sql new file mode 100644 index 000..0fdc615 --- /dev/null +++ b/metastore/scripts/upgrade/mssql/032-HIVE-18202.mssql.sql @@ -0,0 +1,6 @@ +UPDATE TABLE_PARAMS + SET PARAM_KEY = 'hbase.mapreduce.hfileoutputformat.table.name' +WHERE + PARAM_KEY = 'hbase.table.name' +; + http://git-wip-us.apache.org/repos/asf/hive/blob/d37b8026/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index 864a5e5..79189bb 100644 --- a/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -6,6 +6,7 @@ SELECT 'Upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; :r 029-HIVE-16997.mssql.sql :r 030-HIVE-16886.mssql.sql :r 031-HIVE-17566.mssql.sql +:r 032-HIVE-18202.mssql.sql UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; http://git-wip-us.apache.org/repos/asf/hive/blob/d37b8026/metastore/scripts/upgrade/mysql/047-HIVE-18202.mysql.sql -- diff --git a/metastore/scripts/upgrade/mysql/047-HIVE-18202.mysql.sql b/metastore/scripts/upgrade/mysql/047-HIVE-18202.mysql.sql new file mode 100644 index 000..0fdc615 --- /dev/null +++ b/metastore/scripts/upgrade/mysql/047-HIVE-18202.mysql.sql @@ -0,0 +1,6 @@ +UPDATE TABLE_PARAMS + SET PARAM_KEY = '
hive git commit: HIVE-18366: Update HBaseSerDe to use hbase.mapreduce.hfileoutputformat.table.name instead of hbase.table.name as the table name property (Aihua Xu, reviewed Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master 504239869 -> 146234906 HIVE-18366: Update HBaseSerDe to use hbase.mapreduce.hfileoutputformat.table.name instead of hbase.table.name as the table name property (Aihua Xu, reviewed Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/14623490 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/14623490 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/14623490 Branch: refs/heads/master Commit: 146234906982c59953d2dad25c0f1e8978126243 Parents: 5042398 Author: Aihua Xu Authored: Thu Jan 4 12:49:01 2018 -0800 Committer: Aihua Xu Committed: Tue Jan 9 13:54:41 2018 -0800 -- .../apache/hadoop/hive/hbase/HBaseSerDe.java| 2 +- .../src/test/queries/negative/cascade_dbdrop.q | 2 +- .../queries/negative/cascade_dbdrop_hadoop20.q | 2 +- .../src/test/queries/negative/hbase_ddl.q | 2 +- .../test/queries/positive/external_table_ppd.q | 2 +- .../hbase_binary_external_table_queries.q | 6 +-- .../queries/positive/hbase_binary_map_queries.q | 16 +++ .../positive/hbase_binary_map_queries_prefix.q | 6 +-- .../positive/hbase_binary_storage_queries.q | 10 ++--- .../test/queries/positive/hbase_custom_key.q| 4 +- .../test/queries/positive/hbase_custom_key2.q | 2 +- .../test/queries/positive/hbase_custom_key3.q | 2 +- .../src/test/queries/positive/hbase_ddl.q | 2 +- .../src/test/queries/positive/hbase_queries.q | 6 +-- .../queries/positive/hbase_tablename_property.q | 14 ++ .../queries/positive/hbase_timestamp_format.q | 4 +- .../src/test/queries/positive/hbase_viewjoins.q | 4 +- .../test/results/negative/cascade_dbdrop.q.out | 4 +- .../negative/cascade_dbdrop_hadoop20.q.out | 4 +- .../src/test/results/negative/hbase_ddl.q.out | 4 +- .../results/positive/external_table_ppd.q.out | 6 +-- .../hbase_binary_external_table_queries.q.out | 12 ++--- .../positive/hbase_binary_map_queries.q.out | 32 ++--- .../hbase_binary_map_queries_prefix.q.out | 12 ++--- .../positive/hbase_binary_storage_queries.q.out | 24 +- .../results/positive/hbase_custom_key.q.out | 8 ++-- .../results/positive/hbase_custom_key2.q.out| 4 +- .../results/positive/hbase_custom_key3.q.out| 4 +- .../src/test/results/positive/hbase_ddl.q.out | 8 ++-- .../test/results/positive/hbase_queries.q.out | 14 +++--- .../positive/hbase_tablename_property.q.out | 47 .../positive/hbase_timestamp_format.q.out | 8 ++-- .../test/results/positive/hbase_viewjoins.q.out | 8 ++-- .../hive/hcatalog/templeton/TestDesc.java | 2 +- .../hbase/TestPigHBaseStorageHandler.java | 21 +++-- .../hadoop/hive/hbase/HBaseQTestUtil.java | 2 +- 36 files changed, 182 insertions(+), 128 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/14623490/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java -- diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java index c2e7808..1f1bd56 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java @@ -71,7 +71,7 @@ public class HBaseSerDe extends AbstractSerDe { public static final Logger LOG = LoggerFactory.getLogger(HBaseSerDe.class); public static final String HBASE_COLUMNS_MAPPING = "hbase.columns.mapping"; - public static final String HBASE_TABLE_NAME = "hbase.table.name"; + public static final String HBASE_TABLE_NAME = "hbase.mapreduce.hfileoutputformat.table.name"; public static final String HBASE_TABLE_DEFAULT_STORAGE_TYPE = "hbase.table.default.storage.type"; public static final String HBASE_KEY_COL = ":key"; public static final String HBASE_TIMESTAMP_COL = ":timestamp"; http://git-wip-us.apache.org/repos/asf/hive/blob/14623490/hbase-handler/src/test/queries/negative/cascade_dbdrop.q -- diff --git a/hbase-handler/src/test/queries/negative/cascade_dbdrop.q b/hbase-handler/src/test/queries/negative/cascade_dbdrop.q index 7f9df5e..39ecc6b 100644 --- a/hbase-handler/src/test/queries/negative/cascade_dbdrop.q +++ b/hbase-handler/src/test/queries/negative/cascade_dbdrop.q @@ -14,7 +14,7 @@ CREATE DATABASE hbaseDB; CREATE TABLE hbaseDB.hbase_table_0(key int, value string) STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string") -TBLPROPERTIES ("hbase.table.name" = "hbase_table_0"); +TBLPROPERTIES ("hbase.mapreduce.hfileoutputf
[2/3] hive git commit: HIVE-16826: Improvements for SeparatedValuesOutputFormat (BELUGA BEHR, reviewed by Aihua Xu)
HIVE-16826: Improvements for SeparatedValuesOutputFormat (BELUGA BEHR, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0ea2f288 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0ea2f288 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0ea2f288 Branch: refs/heads/master Commit: 0ea2f288a01768a4f431f7a5634b050bc8c3e47a Parents: aa45b02 Author: Aihua Xu Authored: Mon Jan 8 10:46:10 2018 -0800 Committer: Aihua Xu Committed: Mon Jan 8 10:46:10 2018 -0800 -- .../beeline/SeparatedValuesOutputFormat.java| 141 +-- 1 file changed, 70 insertions(+), 71 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/0ea2f288/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java -- diff --git a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java index 9c24a20..172b5b5 100644 --- a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java +++ b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java @@ -22,112 +22,111 @@ */ package org.apache.hive.beeline; -import java.io.IOException; -import java.io.StringWriter; - -import org.apache.hadoop.io.IOUtils; +import org.apache.commons.io.output.StringBuilderWriter; +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; import org.supercsv.encoder.CsvEncoder; +import org.supercsv.encoder.DefaultCsvEncoder; import org.supercsv.encoder.SelectiveCsvEncoder; import org.supercsv.io.CsvListWriter; import org.supercsv.prefs.CsvPreference; /** - * OutputFormat for values separated by a delimiter. + * OutputFormat for values separated by a configurable delimiter */ class SeparatedValuesOutputFormat implements OutputFormat { + + public final static String DSV_OPT_OUTPUT_FORMAT = "dsv"; public final static String DISABLE_QUOTING_FOR_SV = "disable.quoting.for.sv"; + private final static char DEFAULT_QUOTE_CHAR = '"'; private final BeeLine beeLine; - private CsvPreference quotedCsvPreference; - private CsvPreference unquotedCsvPreference; + private final StringBuilderWriter buffer; + private final char defaultSeparator; SeparatedValuesOutputFormat(BeeLine beeLine, char separator) { this.beeLine = beeLine; -unquotedCsvPreference = getUnquotedCsvPreference(separator); -quotedCsvPreference = new CsvPreference.Builder('"', separator, "").build(); +this.defaultSeparator = separator; +this.buffer = new StringBuilderWriter(); } - private static CsvPreference getUnquotedCsvPreference(char delimiter) { -CsvEncoder noEncoder = new SelectiveCsvEncoder(); -return new CsvPreference.Builder('\0', delimiter, "").useEncoder(noEncoder).build(); - } + private CsvPreference getCsvPreference() { +char separator = this.defaultSeparator; +char quoteChar = DEFAULT_QUOTE_CHAR; +CsvEncoder encoder; - private void updateCsvPreference() { -if (beeLine.getOpts().getOutputFormat().equals("dsv")) { - // check whether delimiter changed by user - char curDel = (char) getCsvPreference().getDelimiterChar(); - char newDel = beeLine.getOpts().getDelimiterForDSV(); - // if delimiter changed, rebuild the csv preference - if (newDel != curDel) { -// "" is passed as the end of line symbol in following function, as -// beeline itself adds newline -if (isQuotingDisabled()) { - unquotedCsvPreference = getUnquotedCsvPreference(newDel); -} else { - quotedCsvPreference = new CsvPreference.Builder('"', newDel, "").build(); -} - } +if (DSV_OPT_OUTPUT_FORMAT.equals(beeLine.getOpts().getOutputFormat())) { + separator = beeLine.getOpts().getDelimiterForDSV(); } + +if (isQuotingDisabled()) { + quoteChar = '\0'; + encoder = new SelectiveCsvEncoder(); +} else { + encoder = new DefaultCsvEncoder(); +} + +return new CsvPreference.Builder(quoteChar, separator, StringUtils.EMPTY).useEncoder(encoder).build(); } @Override public int print(Rows rows) { -updateCsvPreference(); - +CsvPreference csvPreference = getCsvPreference(); +CsvListWriter writer = new CsvListWriter(this.buffer, csvPreference); int count = 0; + +Rows.Row labels = (Rows.Row) rows.next(); +if (beeLine.getOpts().getShowHeader()) { + fillBuffer(writer, labels); + String line = getLine(this.buffer); + beeLine.output(line); +} + while (rows.hasNext()) { - if (count == 0 && !beeLine.getOpts().getShowHeader()) { -rows.next(); -count++; -continue; - } -
[1/3] hive git commit: HIVE-17966: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveArrayInspector - Review (BELUGA BEHR, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master a6b88d9d2 -> 148807a98 HIVE-17966: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveArrayInspector - Review (BELUGA BEHR, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aa45b021 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aa45b021 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aa45b021 Branch: refs/heads/master Commit: aa45b021f8f2e65afcb3134ea79c91db86668bcd Parents: a6b88d9 Author: Aihua Xu Authored: Mon Jan 8 10:44:16 2018 -0800 Committer: Aihua Xu Committed: Mon Jan 8 10:44:16 2018 -0800 -- .../serde/ParquetHiveArrayInspector.java| 41 1 file changed, 16 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/aa45b021/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java index 55614a3..c4a286d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java @@ -14,6 +14,7 @@ package org.apache.hadoop.hive.ql.io.parquet.serde; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -69,7 +70,7 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector { } if (data instanceof List) { - return ((List)data).get(index); + return ((List)data).get(index); } throw new UnsupportedOperationException("Cannot inspect " + data.getClass().getCanonicalName()); @@ -91,7 +92,7 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector { } if (data instanceof List) { - return ((List)data).size(); + return ((List)data).size(); } throw new UnsupportedOperationException("Cannot inspect " + data.getClass().getCanonicalName()); @@ -108,13 +109,7 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector { if (array == null) { return null; } - - final List list = new ArrayList(array.length); - for (final Writable obj : array) { -list.add(obj); - } - - return list; + return new ArrayList(Arrays.asList(array)); } if (data instanceof List) { @@ -126,29 +121,27 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector { @Override public Object create(final int size) { -final ArrayList result = new ArrayList(size); -for (int i = 0; i < size; ++i) { - result.add(null); -} -return result; +return new ArrayList(Arrays.asList(new Object[size])); } @Override + @SuppressWarnings("unchecked") public Object set(final Object list, final int index, final Object element) { -final ArrayList l = (ArrayList) list; +final List l = (List) list; l.set(index, element); return list; } @Override + @SuppressWarnings("unchecked") public Object resize(final Object list, final int newSize) { -final ArrayList l = (ArrayList) list; -l.ensureCapacity(newSize); -while (l.size() < newSize) { - l.add(null); -} -while (l.size() > newSize) { - l.remove(l.size() - 1); +final List l = (List) list; +final int deltaSize = newSize - l.size(); +if (deltaSize > 0) { + l.addAll(Arrays.asList(new Object[deltaSize])); +} else { + int size = l.size(); + l.subList(size + deltaSize, size).clear(); } return list; } @@ -167,8 +160,6 @@ public class ParquetHiveArrayInspector implements SettableListObjectInspector { @Override public int hashCode() { -int hash = 3; -hash = 29 * hash + (this.arrayElementInspector != null ? this.arrayElementInspector.hashCode() : 0); -return hash; +return (this.arrayElementInspector != null ? this.arrayElementInspector.hashCode() : 0); } }
[3/3] hive git commit: HIVE-16970: General Improvements To org.apache.hadoop.hive.metastore.cache.CacheUtils (BELUGA BEHR, reviewed by Ashutosh Chauhan)
HIVE-16970: General Improvements To org.apache.hadoop.hive.metastore.cache.CacheUtils (BELUGA BEHR, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/148807a9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/148807a9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/148807a9 Branch: refs/heads/master Commit: 148807a98ad4f0672c9457fff8ba1501c237f1de Parents: 0ea2f28 Author: Aihua Xu Authored: Mon Jan 8 10:48:11 2018 -0800 Committer: Aihua Xu Committed: Mon Jan 8 10:48:11 2018 -0800 -- .../hadoop/hive/metastore/cache/CacheUtils.java | 54 +--- 1 file changed, 23 insertions(+), 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/148807a9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java -- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java index ab6b90f..798ada8 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java @@ -17,11 +17,12 @@ */ package org.apache.hadoop.hive.metastore.cache; -import java.util.ArrayList; -import java.util.HashMap; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.regex.Pattern; +import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -43,14 +44,9 @@ public class CacheUtils { public static String buildKey(String dbName, String tableName, List partVals) { String key = buildKey(dbName, tableName); -if (partVals == null || partVals.size() == 0) { - return key; -} -for (int i = 0; i < partVals.size(); i++) { - key += partVals.get(i); - if (i != partVals.size() - 1) { -key += delimit; - } +if (CollectionUtils.isNotEmpty(partVals)) { + key += delimit; + key += String.join(delimit, partVals); } return key; } @@ -78,28 +74,24 @@ public class CacheUtils { String[] comps = key.split(delimit); result[0] = comps[0]; result[1] = comps[1]; -List vals = new ArrayList<>(); -for (int i=2;i()); + if (sdCopy.getBucketCols() == null) { +sdCopy.setBucketCols(Collections.emptyList()); } - if (sdCopy.getSortCols()==null) { -sdCopy.setSortCols(new ArrayList<>()); + if (sdCopy.getSortCols() == null) { +sdCopy.setSortCols(Collections.emptyList()); } - if (sdCopy.getSkewedInfo()==null) { -sdCopy.setSkewedInfo(new SkewedInfo(new ArrayList<>(), -new ArrayList<>(), new HashMap<>())); + if (sdCopy.getSkewedInfo() == null) { +sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), + Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters()); @@ -110,17 +102,17 @@ public class CacheUtils { static Partition assemble(PartitionWrapper wrapper, SharedCache sharedCache) { Partition p = wrapper.getPartition().deepCopy(); -if (wrapper.getSdHash()!=null) { +if (wrapper.getSdHash() != null) { StorageDescriptor sdCopy = sharedCache.getSdFromCache(wrapper.getSdHash()).deepCopy(); - if (sdCopy.getBucketCols()==null) { -sdCopy.setBucketCols(new ArrayList<>()); + if (sdCopy.getBucketCols() == null) { +sdCopy.setBucketCols(Collections.emptyList()); } - if (sdCopy.getSortCols()==null) { -sdCopy.setSortCols(new ArrayList<>()); + if (sdCopy.getSortCols() == null) { +sdCopy.setSortCols(Collections.emptyList()); } - if (sdCopy.getSkewedInfo()==null) { -sdCopy.setSkewedInfo(new SkewedInfo(new ArrayList<>(), -new ArrayList<>(), new HashMap<>())); + if (sdCopy.getSkewedInfo() == null) { +sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), + Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters());
hive git commit: HIVE-17870: Update NoDeleteRollingFileAppender to use Log4j2 api (Andrew Sheman, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 6a6522a09 -> 1b4baf474 HIVE-17870: Update NoDeleteRollingFileAppender to use Log4j2 api (Andrew Sheman, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1b4baf47 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1b4baf47 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1b4baf47 Branch: refs/heads/master Commit: 1b4baf474c15377cc9f0bacdda317feabeefacaf Parents: 6a6522a Author: Aihua Xu Authored: Fri Dec 1 11:38:34 2017 -0800 Committer: Aihua Xu Committed: Fri Dec 1 13:09:34 2017 -0800 -- pom.xml | 2 +- .../log/HushableRandomAccessFileAppender.java | 3 +- .../ql/log/NoDeleteRollingFileAppender.java | 176 --- .../ql/log/SlidingFilenameRolloverStrategy.java | 82 + .../TestSlidingFilenameRolloverStrategy.java| 145 +++ .../log4j2_test_sliding_rollover.properties | 69 testutils/ptest2/pom.xml| 2 +- 7 files changed, 299 insertions(+), 180 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/1b4baf47/pom.xml -- diff --git a/pom.xml b/pom.xml index 1682f47..6d8ab5e 100644 --- a/pom.xml +++ b/pom.xml @@ -180,7 +180,7 @@ 3.0.3 0.9.3 0.9.3 -2.6.2 +2.8.2 2.3 1.4.1 1.10.19 http://git-wip-us.apache.org/repos/asf/hive/blob/1b4baf47/ql/src/java/org/apache/hadoop/hive/ql/log/HushableRandomAccessFileAppender.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/HushableRandomAccessFileAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/HushableRandomAccessFileAppender.java index 639d1d8..0ff66df 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/log/HushableRandomAccessFileAppender.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/HushableRandomAccessFileAppender.java @@ -176,8 +176,7 @@ public final class HushableRandomAccessFileAppender extends layout = PatternLayout.createDefaultLayout(); } final RandomAccessFileManager manager = RandomAccessFileManager.getFileManager( -fileName, isAppend, isFlush, bufferSize, advertiseURI, layout -// , config -- needed in later log4j versions +fileName, isAppend, isFlush, bufferSize, advertiseURI, layout, config ); if (manager == null) { return null; http://git-wip-us.apache.org/repos/asf/hive/blob/1b4baf47/ql/src/java/org/apache/hadoop/hive/ql/log/NoDeleteRollingFileAppender.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/NoDeleteRollingFileAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/NoDeleteRollingFileAppender.java deleted file mode 100644 index be32f06..000 --- a/ql/src/java/org/apache/hadoop/hive/ql/log/NoDeleteRollingFileAppender.java +++ /dev/null @@ -1,176 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.log; - -import java.io.File; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.io.Writer; - -import org.apache.log4j.FileAppender; -import org.apache.log4j.Layout; -import org.apache.log4j.helpers.CountingQuietWriter; -import org.apache.log4j.helpers.LogLog; -import org.apache.log4j.helpers.OptionConverter; -import org.apache.log4j.spi.LoggingEvent; - -public class NoDeleteRollingFileAppender extends FileAppender { - /** - * The default maximum file size is 10MB. - */ - protected long maxFileSize = 10 * 1024 * 1024; - - private long nextRollover = 0; - - /** - * The default constructor simply calls its {@link FileAppender#FileAppender - * parents constructor}. - */ - public NoDeleteRollingFileAppender() { - } - - /** - * Instantiate a RollingFileAppender and open the file designated by - * filename. The opened filename will b
hive git commit: HIVE-18023: Redact the expression in lineage info (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master d8bc002af -> 83971dec5 HIVE-18023: Redact the expression in lineage info (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/83971dec Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/83971dec Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/83971dec Branch: refs/heads/master Commit: 83971dec5715071414597a2c0043752fe5186ecd Parents: d8bc002 Author: Aihua Xu Authored: Wed Nov 8 14:23:27 2017 -0800 Committer: Aihua Xu Committed: Mon Nov 13 14:45:46 2017 -0800 -- .../java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/83971dec/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java index bdd98cf..2f764f8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java @@ -175,7 +175,7 @@ public class LineageLogger implements ExecuteWithHookContext { List edges = getEdges(plan, index); Set vertices = getVertices(edges); -writeEdges(writer, edges); +writeEdges(writer, edges, hookContext.getConf()); writeVertices(writer, vertices); writer.endObject(); writer.close(); @@ -415,7 +415,8 @@ public class LineageLogger implements ExecuteWithHookContext { /** * Write out an JSON array of edges. */ - private void writeEdges(JsonWriter writer, List edges) throws IOException { + private void writeEdges(JsonWriter writer, List edges, HiveConf conf) + throws IOException, InstantiationException, IllegalAccessException, ClassNotFoundException { writer.name("edges"); writer.beginArray(); for (Edge edge: edges) { @@ -433,7 +434,7 @@ public class LineageLogger implements ExecuteWithHookContext { } writer.endArray(); if (edge.expr != null) { -writer.name("expression").value(edge.expr); +writer.name("expression").value(HookUtils.redactLogString(conf, edge.expr)); } writer.name("edgeType").value(edge.type.name()); writer.endObject();
hive git commit: HIVE-18009: Multiple lateral view query is slow on hive on spark (Aihua Xu, reviewed by Yongzhi Chen & Xuefu Zhang)
Repository: hive Updated Branches: refs/heads/master 51249505c -> d8bc002af HIVE-18009: Multiple lateral view query is slow on hive on spark (Aihua Xu, reviewed by Yongzhi Chen & Xuefu Zhang) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d8bc002a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d8bc002a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d8bc002a Branch: refs/heads/master Commit: d8bc002af63b61e1c1facec1cb5d2f6609377d14 Parents: 5124950 Author: Aihua Xu Authored: Tue Nov 7 16:30:36 2017 -0800 Committer: Aihua Xu Committed: Mon Nov 13 14:41:20 2017 -0800 -- .../test/resources/testconfiguration.properties | 1 + .../hive/ql/parse/spark/GenSparkUtils.java | 30 +++--- .../lateral_view_multi_lateralviews.q | 21 +++ .../lateral_view_multi_lateralviews.q.out | 62 .../spark/lateral_view_multi_lateralviews.q.out | 62 5 files changed, 167 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/d8bc002a/itests/src/test/resources/testconfiguration.properties -- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 5a6dbb8..3b1005f 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -1123,6 +1123,7 @@ spark.query.files=add_part_multiple.q, \ join_vc.q, \ join_view.q, \ lateral_view_explode2.q, \ + lateral_view_multi_lateralviews.q, \ leftsemijoin.q, \ leftsemijoin_mr.q, \ limit_pushdown.q, \ http://git-wip-us.apache.org/repos/asf/hive/blob/d8bc002a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java index 9bea4dd..604c8ae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java @@ -28,6 +28,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Stack; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -617,17 +618,28 @@ public class GenSparkUtils { } @SuppressWarnings("unchecked") - public static T getChildOperator(Operator op, Class klazz) throws SemanticException { -if (klazz.isInstance(op)) { - return (T) op; -} -List> childOperators = op.getChildOperators(); -for (Operator childOp : childOperators) { - T result = getChildOperator(childOp, klazz); - if (result != null) { -return result; + public static T getChildOperator(Operator root, Class klazz) throws SemanticException { +if (root == null) return null; + +HashSet> visited = new HashSet>(); +Stack> stack = new Stack>(); +stack.push(root); +visited.add(root); + +while (!stack.isEmpty()) { + Operator op = stack.pop(); + if (klazz.isInstance(op)) { +return (T) op; + } + List> childOperators = op.getChildOperators(); + for (Operator childOp : childOperators) { +if (!visited.contains(childOp)) { + stack.push(childOp); + visited.add(childOp); +} } } + return null; } http://git-wip-us.apache.org/repos/asf/hive/blob/d8bc002a/ql/src/test/queries/clientpositive/lateral_view_multi_lateralviews.q -- diff --git a/ql/src/test/queries/clientpositive/lateral_view_multi_lateralviews.q b/ql/src/test/queries/clientpositive/lateral_view_multi_lateralviews.q new file mode 100644 index 000..29e026a --- /dev/null +++ b/ql/src/test/queries/clientpositive/lateral_view_multi_lateralviews.q @@ -0,0 +1,21 @@ +CREATE TABLE t1(x5 STRUCT>>> >); +INSERT INTO t1 SELECT NAMED_STRUCT('x4', NAMED_STRUCT('x3', ARRAY(NAMED_STRUCT('x1', 'x1_1', 'x2', ARRAY('x2_1', 'x2_2'); +SELECT c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16 +FROM t1 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c1 +LATERAL VIEW EXPLODE(c1.x2) lv as c2 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c3 +LATERAL VIEW EXPLODE(c1.x2) lv as c4 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c5 +LATERAL VIEW EXPLODE(c1.x2) lv as c6 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c7 +LATERAL VIEW EXPLODE(c1.x2) lv as c8 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c9 +LATERAL VIEW EXPLODE(c1.x2) lv as c10 +LATERAL VIEW EXPLODE(x5.x4.x3) lv as c11 +LATERAL VIEW EXPLODE(c1.x2) lv as c12 +LATERAL VIEW EXPLODE(x5.x4.x
hive git commit: HIVE-17376: Upgrade snappy version to 1.1.4 (Aihua Xu, reviewed by Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/master 3914a1b29 -> 499b13606 HIVE-17376: Upgrade snappy version to 1.1.4 (Aihua Xu, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/499b1360 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/499b1360 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/499b1360 Branch: refs/heads/master Commit: 499b13606a949f5f8525134b66ee798975557eae Parents: 3914a1b Author: Aihua Xu Authored: Thu Nov 9 14:09:09 2017 -0800 Committer: Aihua Xu Committed: Thu Nov 9 14:09:09 2017 -0800 -- pom.xml | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/499b1360/pom.xml -- diff --git a/pom.xml b/pom.xml index 0e9ae3d..dfb29ce 100644 --- a/pom.xml +++ b/pom.xml @@ -201,7 +201,7 @@ 2.11 2.11.8 1.1 -0.2 +1.1.4 1.4 1.5 2.9.1 @@ -938,6 +938,11 @@ jamon-runtime ${jamon-runtime.version} + +org.xerial.snappy +snappy-java +${snappy.version} +
[1/2] hive git commit: HIVE-15016: Run tests with Hadoop 3.0.0-beta1
Repository: hive Updated Branches: refs/heads/master 7ea12e731 -> a3e872822 http://git-wip-us.apache.org/repos/asf/hive/blob/a3e87282/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java -- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java index f8f18b3..120b4af 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java @@ -34,12 +34,16 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; @@ -94,10 +98,17 @@ public class TestPigHBaseStorageHandler extends SkeletonHBaseTest { } - private void populateHBaseTable(String tName) throws IOException { + private void populateHBaseTable(String tName, Connection connection) throws IOException { List myPuts = generatePuts(tName); -HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName)); -table.put(myPuts); +Table table = null; +try { + table = connection.getTable(TableName.valueOf(tName)); + table.put(myPuts); +} finally { + if (table != null) { +table.close(); + } +} } private List generatePuts(String tableName) throws IOException { @@ -107,8 +118,8 @@ public class TestPigHBaseStorageHandler extends SkeletonHBaseTest { myPuts = new ArrayList(); for (int i = 1; i <=10; i++) { Put put = new Put(Bytes.toBytes(i)); - put.add(FAMILY, QUALIFIER1, 1, Bytes.toBytes("textA-" + i)); - put.add(FAMILY, QUALIFIER2, 1, Bytes.toBytes("textB-" + i)); + put.addColumn(FAMILY, QUALIFIER1, 1, Bytes.toBytes("textA-" + i)); + put.addColumn(FAMILY, QUALIFIER2, 1, Bytes.toBytes("textB-" + i)); myPuts.add(put); } return myPuts; @@ -165,8 +176,22 @@ public class TestPigHBaseStorageHandler extends SkeletonHBaseTest { CommandProcessorResponse responseThree = driver.run(tableQuery); -HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); -boolean doesTableExist = hAdmin.tableExists(hbaseTableName); +Connection connection = null; +Admin hAdmin = null; +boolean doesTableExist = false; +try { + connection = ConnectionFactory.createConnection(getHbaseConf()); + hAdmin = connection.getAdmin(); + doesTableExist = hAdmin.tableExists(TableName.valueOf(hbaseTableName)); +} finally { + if (hAdmin != null) { +hAdmin.close(); + } + if (connection != null) { +connection.close(); + } +} + assertTrue(doesTableExist); PigServer server = new PigServer(ExecType.LOCAL,hcatConf.getAllProperties()); @@ -220,17 +245,39 @@ public class TestPigHBaseStorageHandler extends SkeletonHBaseTest { CommandProcessorResponse responseThree = driver.run(tableQuery); -HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); -boolean doesTableExist = hAdmin.tableExists(hbaseTableName); -assertTrue(doesTableExist); +Connection connection = null; +Admin hAdmin = null; +Table table = null; +ResultScanner scanner = null; +boolean doesTableExist = false; +try { + connection = ConnectionFactory.createConnection(getHbaseConf()); + hAdmin = connection.getAdmin(); + doesTableExist = hAdmin.tableExists(TableName.valueOf(hbaseTableName)); + + assertTrue(doesTableExist); + + populateHBaseTable(hbaseTableName, connection); -populateHBaseTable(hbaseTableName); + table = connection.getTable(TableName.valueOf(hbaseTableName)); + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("testFamily")); + scanner = table.getScanner(scan); +} finally { + if (scanner != null) { +scanner.close(); + } + if (table != null ) { +table.close(); + } + if (hAdmin != null) { +hAdmin.close(); + } + if (connection != null) { +connection.close(); + } +} -Configuration conf = new Configuration(getHbaseConf());
[2/2] hive git commit: HIVE-15016: Run tests with Hadoop 3.0.0-beta1
HIVE-15016: Run tests with Hadoop 3.0.0-beta1 Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3e87282 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3e87282 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3e87282 Branch: refs/heads/master Commit: a3e872822ece99eb14a5a12ff8bb191d9e014788 Parents: 7ea12e7 Author: Aihua Xu Authored: Sat Oct 28 16:51:06 2017 -0700 Committer: Aihua Xu Committed: Tue Nov 7 08:39:35 2017 -0800 -- common/pom.xml | 6 + hbase-handler/pom.xml | 45 +++- .../apache/hadoop/hive/hbase/HBaseMetaHook.java | 224 +++ .../hadoop/hive/hbase/HBaseRowSerializer.java | 4 +- .../hadoop/hive/hbase/HBaseStorageHandler.java | 209 ++--- .../hive/hbase/HiveHBaseInputFormatUtil.java| 11 - .../hive/hbase/HiveHBaseTableInputFormat.java | 1 - .../hive/hbase/HiveHBaseTableOutputFormat.java | 21 +- .../hive/hbase/HiveHFileOutputFormat.java | 12 +- .../hadoop/hive/hbase/ResultWritable.java | 5 +- .../hadoop/hive/hbase/TestHBaseSerDe.java | 120 +- .../hadoop/hive/hbase/TestLazyHBaseObject.java | 79 +++ .../hive/hbase/TestPutResultWritable.java | 7 +- .../src/test/queries/positive/hbase_bulk.q | 2 +- .../test/queries/positive/hbase_handler_bulk.q | 2 +- .../results/positive/hbase_handler_bulk.q.out | 4 +- .../apache/hive/hcatalog/common/HCatUtil.java | 2 +- .../rcfile/TestRCFileMapReduceInputFormat.java | 14 +- .../hive/hcatalog/templeton/TestWebHCatE2e.java | 4 +- .../hcatalog/templeton/mock/MockUriInfo.java| 11 + itests/hcatalog-unit/pom.xml| 6 + .../hive/hcatalog/hbase/ManyMiniCluster.java| 27 ++- .../hive/hcatalog/hbase/SkeletonHBaseTest.java | 24 +- .../hbase/TestPigHBaseStorageHandler.java | 206 +++-- itests/hive-minikdc/pom.xml | 8 +- itests/hive-unit-hadoop2/pom.xml| 6 + itests/hive-unit/pom.xml| 31 ++- .../apache/hadoop/hive/ql/TestAcidOnTez.java| 3 + .../hive/ql/txn/compactor/TestCompactor.java| 6 +- .../jdbc/TestJdbcWithLocalClusterSpark.java | 5 + ...stMultiSessionsHS2WithLocalClusterSpark.java | 5 + itests/qtest-accumulo/pom.xml | 37 ++- itests/qtest-spark/pom.xml | 21 +- itests/qtest/pom.xml| 6 + itests/util/pom.xml | 19 +- .../hadoop/hive/hbase/HBaseQTestUtil.java | 37 +-- .../hadoop/hive/hbase/HBaseTestSetup.java | 46 ++-- llap-server/pom.xml | 61 - .../llap/shufflehandler/ShuffleHandler.java | 26 +-- metastore/pom.xml | 2 +- pom.xml | 77 ++- .../apache/hadoop/hive/ql/io/TestRCFile.java| 17 +- .../exim_00_unsupported_schema.q.out| 2 +- .../test/results/clientnegative/external1.q.out | 2 +- .../test/results/clientnegative/external2.q.out | 2 +- serde/pom.xml | 6 + shims/0.23/pom.xml | 12 + .../apache/hadoop/hive/shims/Hadoop23Shims.java | 43 ++-- .../org/apache/hadoop/fs/ProxyFileSystem.java | 12 + shims/scheduler/pom.xml | 8 +- spark-client/pom.xml| 4 + standalone-metastore/pom.xml| 18 +- .../hadoop/hive/metastore/utils/HdfsUtils.java | 10 +- .../ptest2/src/main/resources/batch-exec.vm | 2 +- 54 files changed, 1027 insertions(+), 553 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/a3e87282/common/pom.xml -- diff --git a/common/pom.xml b/common/pom.xml index 868e14d..aaeecc0 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -220,6 +220,12 @@ io.dropwizard.metrics metrics-json ${dropwizard.version} + + + com.fasterxml.jackson.core + jackson-databind + + com.fasterxml.jackson.core http://git-wip-us.apache.org/repos/asf/hive/blob/a3e87282/hbase-handler/pom.xml -- diff --git a/hbase-handler/pom.xml b/hbase-handler/pom.xml index 7f57b77..59f9dd8 100644 --- a/hbase-handler/pom.xml +++ b/hbase-handler/pom.xml @@ -81,8 +81,23 @@ org.apache.hbase hbase-server ${hbase.version} - - + + +org.slf4j +slf4j-log4j12 + + +commmons-logging +commons-logging + +
hive git commit: HIVE-17826: Error writing to RandomAccessFile after operation log is closed (Andrew Sherman, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master df321c843 -> b57837abc HIVE-17826: Error writing to RandomAccessFile after operation log is closed (Andrew Sherman, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b57837ab Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b57837ab Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b57837ab Branch: refs/heads/master Commit: b57837abc0014a11d0db988b118ce266352ad1d2 Parents: df321c8 Author: Aihua Xu Authored: Mon Oct 30 13:39:12 2017 -0700 Committer: Aihua Xu Committed: Mon Oct 30 13:42:43 2017 -0700 -- .../operation/TestOperationLoggingLayout.java | 26 +++ .../log/HushableRandomAccessFileAppender.java | 191 +++ .../hadoop/hive/ql/log/LogDivertAppender.java | 20 +- .../hive/ql/log/LogDivertAppenderForTest.java | 19 +- 4 files changed, 233 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/b57837ab/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java index 3c30069..8febe3e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.log.HushableRandomAccessFileAppender; import org.apache.hadoop.hive.ql.log.LogDivertAppender; import org.apache.hadoop.hive.ql.log.LogDivertAppenderForTest; import org.apache.hive.jdbc.miniHS2.MiniHS2; @@ -34,7 +35,9 @@ import org.apache.hive.service.cli.FetchType; import org.apache.hive.service.cli.OperationHandle; import org.apache.hive.service.cli.RowSet; import org.apache.hive.service.cli.SessionHandle; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.AbstractLogEvent; import org.apache.logging.log4j.core.Appender; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.routing.RoutingAppender; @@ -152,6 +155,16 @@ public class TestOperationLoggingLayout { Appender appender = appenderControl.getAppender(); Assert.assertNotNull(msg + "could not find Appender for query id " + queryId + " from AppenderControl " + appenderControl, appender); Assert.assertEquals(msg + "Appender for query is in unexpected state", expectedStopped, appender.isStopped()); +Assert.assertTrue("Appender should be a HushableMutableRandomAccessAppender", appender instanceof HushableRandomAccessFileAppender); +HushableRandomAccessFileAppender ra = (HushableRandomAccessFileAppender) appender; +// Even if the appender is stopped it should not throw an exception when we log +try { + ra.append(new LocalLogEvent()); +} catch (Exception e) { + e.printStackTrace(); + Assert.fail("Caught exception while logging to an appender of class " + ra.getClass() + + " with stopped=" + ra.isStopped()); +} } private SessionHandle setupSession() throws Exception { @@ -184,4 +197,17 @@ public class TestOperationLoggingLayout { return sessionHandle; } + + /** + * A minimal LogEvent implementation for testing + */ + private static class LocalLogEvent extends AbstractLogEvent { + +LocalLogEvent() { +} + +@Override public Level getLevel() { + return Level.DEBUG; +} + } } http://git-wip-us.apache.org/repos/asf/hive/blob/b57837ab/ql/src/java/org/apache/hadoop/hive/ql/log/HushableRandomAccessFileAppender.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/HushableRandomAccessFileAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/HushableRandomAccessFileAppender.java new file mode 100644 index 000..639d1d8 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/HushableRandomAccessFileAppender.java @@ -0,0 +1,191 @@ +package org.apache.hadoop.hive.ql.log; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use
hive git commit: HIVE-17831: HiveSemanticAnalyzerHookContext does not update the HiveOperation after sem.analyze() is called (Aihua Xu, reviewed by Sergio Pena)
Repository: hive Updated Branches: refs/heads/branch-2.1 6db9fd6e4 -> f9bd36004 HIVE-17831: HiveSemanticAnalyzerHookContext does not update the HiveOperation after sem.analyze() is called (Aihua Xu, reviewed by Sergio Pena) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f9bd3600 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f9bd3600 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f9bd3600 Branch: refs/heads/branch-2.1 Commit: f9bd360042311ef6677a6a1c879eb3d1bf59a95f Parents: 6db9fd6 Author: Aihua Xu Authored: Wed Oct 18 17:01:20 2017 -0700 Committer: Aihua Xu Committed: Fri Oct 20 09:55:53 2017 -0700 -- .../apache/hadoop/hive/hooks/TestHs2Hooks.java | 36 .../HiveSemanticAnalyzerHookContextImpl.java| 1 + 2 files changed, 37 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/f9bd3600/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java index dad516c..7f2517b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java @@ -187,6 +187,7 @@ public class TestHs2Hooks { Properties connProp = new Properties(); connProp.setProperty("user", System.getProperty("user.name")); connProp.setProperty("password", ""); + HiveConnection connection = new HiveConnection("jdbc:hive2://localhost:1/default", connProp); Statement stmt = connection.createStatement(); stmt.executeQuery("show databases"); @@ -234,5 +235,40 @@ public class TestHs2Hooks { Assert.assertTrue(SemanticAnalysisHook.ipAddress, SemanticAnalysisHook.ipAddress.contains("127.0.0.1")); Assert.assertEquals("show tables", SemanticAnalysisHook.command); + +stmt.close(); +connection.close(); + } + + @Test + public void testPostAnalysisHookContexts() throws Throwable { +Properties connProp = new Properties(); +connProp.setProperty("user", System.getProperty("user.name")); +connProp.setProperty("password", ""); + +HiveConnection connection = new HiveConnection("jdbc:hive2://localhost:1/default", connProp); +Statement stmt = connection.createStatement(); +stmt.execute("create table testPostAnalysisHookContexts as select '3'"); +Throwable error = PostExecHook.error; +if (error != null) { + throw error; +} +error = PreExecHook.error; +if (error != null) { + throw error; +} + +Assert.assertEquals(HiveOperation.CREATETABLE_AS_SELECT, SemanticAnalysisHook.commandType); + +error = SemanticAnalysisHook.preAnalyzeError; +if (error != null) { + throw error; +} +error = SemanticAnalysisHook.postAnalyzeError; +if (error != null) { + throw error; +} +stmt.close(); +connection.close(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/f9bd3600/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java index 1cc38a8..e28dc47 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java @@ -58,6 +58,7 @@ public class HiveSemanticAnalyzerHookContextImpl implements HiveSemanticAnalyzer public void update(BaseSemanticAnalyzer sem) { this.inputs = sem.getInputs(); this.outputs = sem.getOutputs(); +this.commandType = sem.getQueryState().getHiveOperation(); } @Override
hive git commit: HIVE-17831: HiveSemanticAnalyzerHookContext does not update the HiveOperation after sem.analyze() is called (Aihua Xu, reviewed by Sergio Pena)
Repository: hive Updated Branches: refs/heads/branch-2.2 fbbffda3a -> 43d8647fd HIVE-17831: HiveSemanticAnalyzerHookContext does not update the HiveOperation after sem.analyze() is called (Aihua Xu, reviewed by Sergio Pena) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/43d8647f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/43d8647f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/43d8647f Branch: refs/heads/branch-2.2 Commit: 43d8647fd0dbe350638478a7f9e8415cead813d2 Parents: fbbffda Author: Aihua Xu Authored: Wed Oct 18 17:01:20 2017 -0700 Committer: Aihua Xu Committed: Fri Oct 20 09:55:02 2017 -0700 -- .../apache/hadoop/hive/hooks/TestHs2Hooks.java | 36 .../HiveSemanticAnalyzerHookContextImpl.java| 1 + 2 files changed, 37 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/43d8647f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java index dad516c..7f2517b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java @@ -187,6 +187,7 @@ public class TestHs2Hooks { Properties connProp = new Properties(); connProp.setProperty("user", System.getProperty("user.name")); connProp.setProperty("password", ""); + HiveConnection connection = new HiveConnection("jdbc:hive2://localhost:1/default", connProp); Statement stmt = connection.createStatement(); stmt.executeQuery("show databases"); @@ -234,5 +235,40 @@ public class TestHs2Hooks { Assert.assertTrue(SemanticAnalysisHook.ipAddress, SemanticAnalysisHook.ipAddress.contains("127.0.0.1")); Assert.assertEquals("show tables", SemanticAnalysisHook.command); + +stmt.close(); +connection.close(); + } + + @Test + public void testPostAnalysisHookContexts() throws Throwable { +Properties connProp = new Properties(); +connProp.setProperty("user", System.getProperty("user.name")); +connProp.setProperty("password", ""); + +HiveConnection connection = new HiveConnection("jdbc:hive2://localhost:1/default", connProp); +Statement stmt = connection.createStatement(); +stmt.execute("create table testPostAnalysisHookContexts as select '3'"); +Throwable error = PostExecHook.error; +if (error != null) { + throw error; +} +error = PreExecHook.error; +if (error != null) { + throw error; +} + +Assert.assertEquals(HiveOperation.CREATETABLE_AS_SELECT, SemanticAnalysisHook.commandType); + +error = SemanticAnalysisHook.preAnalyzeError; +if (error != null) { + throw error; +} +error = SemanticAnalysisHook.postAnalyzeError; +if (error != null) { + throw error; +} +stmt.close(); +connection.close(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/43d8647f/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java index 1cc38a8..e28dc47 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java @@ -58,6 +58,7 @@ public class HiveSemanticAnalyzerHookContextImpl implements HiveSemanticAnalyzer public void update(BaseSemanticAnalyzer sem) { this.inputs = sem.getInputs(); this.outputs = sem.getOutputs(); +this.commandType = sem.getQueryState().getHiveOperation(); } @Override
hive git commit: HIVE-17831: HiveSemanticAnalyzerHookContext does not update the HiveOperation after sem.analyze() is called (Aihua Xu, reviewed by Sergio Pena)
Repository: hive Updated Branches: refs/heads/branch-2 2106bf777 -> 417a7a654 HIVE-17831: HiveSemanticAnalyzerHookContext does not update the HiveOperation after sem.analyze() is called (Aihua Xu, reviewed by Sergio Pena) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/417a7a65 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/417a7a65 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/417a7a65 Branch: refs/heads/branch-2 Commit: 417a7a6549370b22fc965751ed5602376db447c0 Parents: 2106bf7 Author: Aihua Xu Authored: Wed Oct 18 17:01:20 2017 -0700 Committer: Aihua Xu Committed: Fri Oct 20 09:47:57 2017 -0700 -- .../apache/hadoop/hive/hooks/TestHs2Hooks.java | 36 .../HiveSemanticAnalyzerHookContextImpl.java| 1 + 2 files changed, 37 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/417a7a65/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java index dad516c..7f2517b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java @@ -187,6 +187,7 @@ public class TestHs2Hooks { Properties connProp = new Properties(); connProp.setProperty("user", System.getProperty("user.name")); connProp.setProperty("password", ""); + HiveConnection connection = new HiveConnection("jdbc:hive2://localhost:1/default", connProp); Statement stmt = connection.createStatement(); stmt.executeQuery("show databases"); @@ -234,5 +235,40 @@ public class TestHs2Hooks { Assert.assertTrue(SemanticAnalysisHook.ipAddress, SemanticAnalysisHook.ipAddress.contains("127.0.0.1")); Assert.assertEquals("show tables", SemanticAnalysisHook.command); + +stmt.close(); +connection.close(); + } + + @Test + public void testPostAnalysisHookContexts() throws Throwable { +Properties connProp = new Properties(); +connProp.setProperty("user", System.getProperty("user.name")); +connProp.setProperty("password", ""); + +HiveConnection connection = new HiveConnection("jdbc:hive2://localhost:1/default", connProp); +Statement stmt = connection.createStatement(); +stmt.execute("create table testPostAnalysisHookContexts as select '3'"); +Throwable error = PostExecHook.error; +if (error != null) { + throw error; +} +error = PreExecHook.error; +if (error != null) { + throw error; +} + +Assert.assertEquals(HiveOperation.CREATETABLE_AS_SELECT, SemanticAnalysisHook.commandType); + +error = SemanticAnalysisHook.preAnalyzeError; +if (error != null) { + throw error; +} +error = SemanticAnalysisHook.postAnalyzeError; +if (error != null) { + throw error; +} +stmt.close(); +connection.close(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/417a7a65/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java index 1cc38a8..e28dc47 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java @@ -58,6 +58,7 @@ public class HiveSemanticAnalyzerHookContextImpl implements HiveSemanticAnalyzer public void update(BaseSemanticAnalyzer sem) { this.inputs = sem.getInputs(); this.outputs = sem.getOutputs(); +this.commandType = sem.getQueryState().getHiveOperation(); } @Override
hive git commit: HIVE-17831: HiveSemanticAnalyzerHookContext does not update the HiveOperation after sem.analyze() is called (Aihua Xu, reviewed by Sergio Pena)
Repository: hive Updated Branches: refs/heads/master 76d880937 -> b9bc82258 HIVE-17831: HiveSemanticAnalyzerHookContext does not update the HiveOperation after sem.analyze() is called (Aihua Xu, reviewed by Sergio Pena) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b9bc8225 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b9bc8225 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b9bc8225 Branch: refs/heads/master Commit: b9bc822585e9c7482f93dfb71f1769a0f009245d Parents: 76d8809 Author: Aihua Xu Authored: Wed Oct 18 17:01:20 2017 -0700 Committer: Aihua Xu Committed: Fri Oct 20 09:36:33 2017 -0700 -- .../apache/hadoop/hive/hooks/TestHs2Hooks.java | 36 .../HiveSemanticAnalyzerHookContextImpl.java| 1 + 2 files changed, 37 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/b9bc8225/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java index dad516c..7f2517b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java @@ -187,6 +187,7 @@ public class TestHs2Hooks { Properties connProp = new Properties(); connProp.setProperty("user", System.getProperty("user.name")); connProp.setProperty("password", ""); + HiveConnection connection = new HiveConnection("jdbc:hive2://localhost:1/default", connProp); Statement stmt = connection.createStatement(); stmt.executeQuery("show databases"); @@ -234,5 +235,40 @@ public class TestHs2Hooks { Assert.assertTrue(SemanticAnalysisHook.ipAddress, SemanticAnalysisHook.ipAddress.contains("127.0.0.1")); Assert.assertEquals("show tables", SemanticAnalysisHook.command); + +stmt.close(); +connection.close(); + } + + @Test + public void testPostAnalysisHookContexts() throws Throwable { +Properties connProp = new Properties(); +connProp.setProperty("user", System.getProperty("user.name")); +connProp.setProperty("password", ""); + +HiveConnection connection = new HiveConnection("jdbc:hive2://localhost:1/default", connProp); +Statement stmt = connection.createStatement(); +stmt.execute("create table testPostAnalysisHookContexts as select '3'"); +Throwable error = PostExecHook.error; +if (error != null) { + throw error; +} +error = PreExecHook.error; +if (error != null) { + throw error; +} + +Assert.assertEquals(HiveOperation.CREATETABLE_AS_SELECT, SemanticAnalysisHook.commandType); + +error = SemanticAnalysisHook.preAnalyzeError; +if (error != null) { + throw error; +} +error = SemanticAnalysisHook.postAnalyzeError; +if (error != null) { + throw error; +} +stmt.close(); +connection.close(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/b9bc8225/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java index 1cc38a8..e28dc47 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java @@ -58,6 +58,7 @@ public class HiveSemanticAnalyzerHookContextImpl implements HiveSemanticAnalyzer public void update(BaseSemanticAnalyzer sem) { this.inputs = sem.getInputs(); this.outputs = sem.getOutputs(); +this.commandType = sem.getQueryState().getHiveOperation(); } @Override
hive git commit: HIVE-17679: http-generic-click-jacking for WebHcat server (Aihua Xu reviewed by Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master 26753ade2 -> 2902c7cc2 HIVE-17679: http-generic-click-jacking for WebHcat server (Aihua Xu reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2902c7cc Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2902c7cc Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2902c7cc Branch: refs/heads/master Commit: 2902c7cc2ab20525139cafa8c594a09fb6c499f9 Parents: 26753ad Author: Aihua Xu Authored: Tue Oct 3 09:44:07 2017 -0700 Committer: Aihua Xu Committed: Thu Oct 5 14:59:14 2017 -0700 -- .../svr/src/main/config/webhcat-default.xml | 8 .../hive/hcatalog/templeton/AppConfig.java | 1 + .../apache/hive/hcatalog/templeton/Main.java| 43 3 files changed, 52 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/2902c7cc/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml -- diff --git a/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml b/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml index fa8dbf8..2de8525 100644 --- a/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml +++ b/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml @@ -371,4 +371,12 @@ in all PUT/POST requests, and rejects requests that do not have these. + +templeton.frame.options.filter +DENY + +X-Frame-Options is added in HTTP response header with this value to prevent +clickjacking attacks. Possible values are DENY, SAMEORIGIN, ALLOW-FROM uri. + + http://git-wip-us.apache.org/repos/asf/hive/blob/2902c7cc/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java -- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java index 0ea7d88..4232d4d 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java @@ -204,6 +204,7 @@ public class AppConfig extends Configuration { public static final String HIVE_EXTRA_FILES = "templeton.hive.extra.files"; public static final String XSRF_FILTER_ENABLED = "templeton.xsrf.filter.enabled"; + public static final String FRAME_OPTIONS_FILETER = "templeton.frame.options.filter"; private static final Logger LOG = LoggerFactory.getLogger(AppConfig.class); http://git-wip-us.apache.org/repos/asf/hive/blob/2902c7cc/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java -- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java index 3ed3ece..02b9db9 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java @@ -53,7 +53,15 @@ import org.eclipse.jetty.xml.XmlConfiguration; import org.slf4j.bridge.SLF4JBridgeHandler; import javax.servlet.DispatcherType; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.annotation.WebFilter; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; /** * The main executable that starts up and runs the Server. @@ -213,6 +221,8 @@ public class Main { LOG.warn("XSRF filter disabled"); } +root.addFilter(makeFrameOptionFilter(), "/" + SERVLET_PATH + "/*", dispatches); + // Connect Jersey ServletHolder h = new ServletHolder(new ServletContainer(makeJerseyConfig())); root.addServlet(h, "/" + SERVLET_PATH + "/*"); @@ -259,6 +269,39 @@ public class Main { return authFilter; } + public FilterHolder makeFrameOptionFilter() { +FilterHolder frameOptionFilter = new FilterHolder(XFrameOptionsFilter.class); +frameOptionFilter.setInitParameter(AppConfig.FRAME_OPTIONS_FILETER, conf.get(AppConfig.FRAME_OPTIONS_FILETER)); +return frameOptionFilter; + } + + public static class XFrameOptionsFilter implements Filter { +private final static String defaultMode = "DENY"; + +private String mode = null; + +@Override +public void init(FilterConfig filterConfig) throws ServletException { + mode = filterConfig
hive git commit: HIVE-17544: Provide classname info for function authorization (Aihua Xu, reviewed by Sergio Pena)
Repository: hive Updated Branches: refs/heads/master addeab8d0 -> 31077be9b HIVE-17544: Provide classname info for function authorization (Aihua Xu, reviewed by Sergio Pena) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/31077be9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/31077be9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/31077be9 Branch: refs/heads/master Commit: 31077be9b90832acc5eb1641690955945ed5a3a1 Parents: addeab8 Author: Aihua Xu Authored: Fri Sep 29 15:57:27 2017 -0700 Committer: Aihua Xu Committed: Wed Oct 4 13:06:04 2017 -0700 -- .../java/org/apache/hadoop/hive/ql/Driver.java | 4 +++- .../org/apache/hadoop/hive/ql/hooks/Entity.java | 17 - .../hadoop/hive/ql/hooks/WriteEntity.java | 5 ++-- .../hive/ql/parse/FunctionSemanticAnalyzer.java | 9 --- .../plugin/HiveAuthorizerImpl.java | 1 - .../plugin/HivePrivilegeObject.java | 25 6 files changed, 46 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/31077be9/ql/src/java/org/apache/hadoop/hive/ql/Driver.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index f01edf8..1943c6d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -1103,6 +1103,7 @@ public class Driver implements CommandProcessor { String objName = null; List partKeys = null; List columns = null; + String className = null; switch(privObject.getType()){ case DATABASE: dbname = privObject.getDatabase().getName(); @@ -1122,6 +1123,7 @@ public class Driver implements CommandProcessor { dbname = privObject.getDatabase().getName(); } objName = privObject.getFunctionName(); +className = privObject.getClassName(); break; case DUMMYPARTITION: case PARTITION: @@ -1135,7 +1137,7 @@ public class Driver implements CommandProcessor { } HivePrivObjectActionType actionType = AuthorizationUtils.getActionType(privObject); HivePrivilegeObject hPrivObject = new HivePrivilegeObject(privObjType, dbname, objName, - partKeys, columns, actionType, null); + partKeys, columns, actionType, null, className); hivePrivobjs.add(hPrivObject); } return hivePrivobjs; http://git-wip-us.apache.org/repos/asf/hive/blob/31077be9/ql/src/java/org/apache/hadoop/hive/ql/hooks/Entity.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/Entity.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/Entity.java index 820e4e2..c3c4512 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/Entity.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/Entity.java @@ -76,6 +76,11 @@ public class Entity implements Serializable { private String stringObject; /** + * The class name for a function + */ + private String className; + + /** * This is derived from t and p, but we need to serialize this field to make * sure Entity.hashCode() does not need to recursively read into t and p. */ @@ -139,6 +144,14 @@ public class Entity implements Serializable { this.d = d; } + public String getClassName() { +return this.className; + } + + public void setClassName(String className) { +this.className = className; + } + public String getFunctionName() { if (typ == Type.FUNCTION) { return stringObject; @@ -254,15 +267,17 @@ public class Entity implements Serializable { * Create an entity representing a object with given name, database namespace and type * @param database - database namespace * @param strObj - object name as string + * @param className - function class name * @param type - the entity type. this constructor only supports FUNCTION type currently */ - public Entity(Database database, String strObj, Type type) { + public Entity(Database database, String strObj, String className, Type type) { if (type != Type.FUNCTION) { throw new IllegalArgumentException("This constructor is supported only for type:" + Type.FUNCTION); } this.database = database; this.stringObject = strObj; +this.className = className; this.typ = type; this.complete = true; name = computeName(); http://git-wip-us.apache.org/repos/asf/hive/blob/31077be9/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hoo
[2/2] hive git commit: HIVE-17619: Exclude avatica-core.jar dependency from avatica shaded jar (Aihua Xu, reviewed by Yongzhi Chen)
HIVE-17619: Exclude avatica-core.jar dependency from avatica shaded jar (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2216dad5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2216dad5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2216dad5 Branch: refs/heads/master Commit: 2216dad543c8a4f5ee732b386fb7c39ab48d0760 Parents: 8382008 Author: Aihua Xu Authored: Wed Sep 27 09:51:28 2017 -0700 Committer: Aihua Xu Committed: Fri Sep 29 10:58:25 2017 -0700 -- ql/pom.xml | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/2216dad5/ql/pom.xml -- diff --git a/ql/pom.xml b/ql/pom.xml index 3dfb421..59c4260 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -402,6 +402,10 @@ avatica ${avatica.version} + + org.apache.calcite.avatica + avatica-core +
[1/2] hive git commit: HIVE-17624: MapredLocakTask running in separate JVM could throw ClassNotFoundException (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master 11beadff3 -> 2216dad54 HIVE-17624: MapredLocakTask running in separate JVM could throw ClassNotFoundException (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8382008e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8382008e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8382008e Branch: refs/heads/master Commit: 8382008ee00d8910b920a0c7a6d73f2f3400cb8a Parents: 11beadf Author: Aihua Xu Authored: Wed Sep 27 14:12:27 2017 -0700 Committer: Aihua Xu Committed: Fri Sep 29 09:48:25 2017 -0700 -- .../hadoop/hive/ql/exec/mr/MapredLocalTask.java |7 +- .../queries/clientpositive/mapjoin_addjar.q | 13 +- .../results/clientpositive/mapjoin_addjar.q.out | 1045 +- .../clientpositive/spark/mapjoin_addjar.q.out | 1045 +- 4 files changed, 2079 insertions(+), 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/8382008e/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java index c5d4f9a..9dfefee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java @@ -71,6 +71,7 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -184,12 +185,12 @@ public class MapredLocalTask extends Task implements Serializab IOUtils.closeQuietly(out); } - String isSilent = "true".equalsIgnoreCase(System.getProperty("test.silent")) ? "-nolog" : ""; - String jarCmd; + String libJars = ExecDriver.getResource(conf, ResourceType.JAR); + String libJarsOption = StringUtils.isEmpty(libJars) ? " " : " -libjars " + libJars + " "; - jarCmd = hiveJar + " " + ExecDriver.class.getName(); + String jarCmd = hiveJar + " " + ExecDriver.class.getName() + libJarsOption; String hiveConfArgs = ExecDriver.generateCmdLine(conf, ctx); String cmdLine = hadoopExec + " jar " + jarCmd + " -localtask -plan " + planPath.toString() + " " + isSilent + " " + hiveConfArgs; http://git-wip-us.apache.org/repos/asf/hive/blob/8382008e/ql/src/test/queries/clientpositive/mapjoin_addjar.q -- diff --git a/ql/src/test/queries/clientpositive/mapjoin_addjar.q b/ql/src/test/queries/clientpositive/mapjoin_addjar.q index f56f074..1d71512 100644 --- a/ql/src/test/queries/clientpositive/mapjoin_addjar.q +++ b/ql/src/test/queries/clientpositive/mapjoin_addjar.q @@ -2,13 +2,14 @@ set hive.auto.convert.join=true; set hive.auto.convert.join.use.nonstaged=false; -add jar ${system:maven.local.repository}/org/apache/hive/hcatalog/hive-hcatalog-core/${system:hive.version}/hive-hcatalog-core-${system:hive.version}.jar; +ADD JAR ${system:maven.local.repository}/org/apache/hive/hive-it-test-serde/${system:hive.version}/hive-it-test-serde-${system:hive.version}.jar; + +CREATE TABLE t1(KEY STRING, VALUE STRING) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.TestSerDe' STORED AS TEXTFILE; +LOAD DATA LOCAL INPATH '../../data/files/kv1_cb.txt' INTO TABLE t1; + +select * from t1 l join t1 r on l.key =r.key; -CREATE TABLE t1 (a string, b string) -ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' -; -LOAD DATA LOCAL INPATH "../../data/files/sample.json" INTO TABLE t1; -select * from src join t1 on src.key =t1.a; drop table t1; +DELETE JAR ${system:maven.local.repository}/org/apache/hive/hive-it-test-serde/${system:hive.version}/hive-it-test-serde-${system:hive.version}.jar; set hive.auto.convert.join=false; http://git-wip-us.apache.org/repos/asf/hive/blob/8382008e/ql/src/test/results/clientpositive/mapjoin_addjar.q.out -- diff --git a/ql/src/test/results/clientpositive/mapjoin_addjar.q.out b/ql/src/test/results/clientpositive/mapjoin_addjar.q.out index c107818..c73b985 100644 --- a/ql/src/test/results/clientpositive/mapjoin_addjar.q.out +++ b/ql/src/test/results/clientpositive/mapjoin_addjar.q.out @@ -1,32 +1,1055 @@ -PREHOOK:
hive git commit: Revert "HIVE-17373: Upgrade some dependency versions (Aihua Xu, reviewed by Naveen Gangam)"
Repository: hive Updated Branches: refs/heads/master 09c60fd3a -> ccd02c837 Revert "HIVE-17373: Upgrade some dependency versions (Aihua Xu, reviewed by Naveen Gangam)" This reverts commit 5f13f286cfddc4ed0a06eb40b7c69a5ca69ab300. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ccd02c83 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ccd02c83 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ccd02c83 Branch: refs/heads/master Commit: ccd02c83780c938069edec179d8d06500b80e779 Parents: 09c60fd Author: Aihua Xu Authored: Tue Sep 26 13:52:31 2017 -0700 Committer: Aihua Xu Committed: Tue Sep 26 13:52:31 2017 -0700 -- .../mapjoin/MapJoinMultiKeyBench.java | 94 +++- .../mapjoin/MapJoinOneLongKeyBench.java | 94 +++- .../mapjoin/MapJoinOneStringKeyBench.java | 94 +++- pom.xml | 6 +- .../hadoop/hive/ql/log/LogDivertAppender.java | 25 +++--- .../hive/ql/log/LogDivertAppenderForTest.java | 18 ++-- 6 files changed, 304 insertions(+), 27 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/ccd02c83/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java -- diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java index 7a5b721..f183bb5 100644 --- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java +++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java @@ -18,15 +18,107 @@ package org.apache.hive.benchmark.vectorization.mapjoin; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorBatchDebug; +import org.apache.hadoop.hive.ql.exec.vector.VectorColumnOutputMapping; +import org.apache.hadoop.hive.ql.exec.vector.VectorColumnSourceMapping; +import org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow; +import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOuterFilteredOperator; +import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource; +import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; +import org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator; +import org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator.GenerateType; +import org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator.GenerateType.GenerateCategory; +import org.apache.hadoop.hive.ql.exec.vector.expressions.ColAndCol; +import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig.MapJoinTestImplementation; +import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestData; +import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestDescription; +import org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.VectorMapJoinFastMultiKeyHashMap; +import org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.VectorMapJoinFastTableContainer; +import org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.VerifyFastRow; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.JoinCond
hive git commit: HIVE-17429: Hive JDBC doesn't return rows when querying Impala (Zach Amsden, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master c52aba1a6 -> 8482c5fbe HIVE-17429: Hive JDBC doesn't return rows when querying Impala (Zach Amsden, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8482c5fb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8482c5fb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8482c5fb Branch: refs/heads/master Commit: 8482c5fbe9d2b9a62132e0e94d5578a5eaa22fbd Parents: c52aba1 Author: Aihua Xu Authored: Thu Sep 7 17:17:14 2017 -0700 Committer: Aihua Xu Committed: Thu Sep 7 17:17:14 2017 -0700 -- jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/8482c5fb/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java -- diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java index b743b46..c6bd41f 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java @@ -263,7 +263,7 @@ public class HiveStatement implements java.sql.Statement { TGetOperationStatusResp status = waitForOperationToComplete(); // The query should be completed by now -if (!status.isHasResultSet()) { +if (!status.isHasResultSet() && !stmtHandle.isHasResultSet()) { return false; } resultSet = new HiveQueryResultSet.Builder(this).setClient(client).setSessionHandle(sessHandle)
hive git commit: HIVE-17373: Upgrade some dependency versions (Aihua Xu, reviewed by Naveen Gangam)
Repository: hive Updated Branches: refs/heads/master 9a09b78b6 -> 5f13f286c HIVE-17373: Upgrade some dependency versions (Aihua Xu, reviewed by Naveen Gangam) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5f13f286 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5f13f286 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5f13f286 Branch: refs/heads/master Commit: 5f13f286cfddc4ed0a06eb40b7c69a5ca69ab300 Parents: 9a09b78 Author: Aihua Xu Authored: Tue Aug 22 16:05:58 2017 -0700 Committer: Aihua Xu Committed: Tue Sep 5 11:23:49 2017 -0700 -- .../mapjoin/MapJoinMultiKeyBench.java | 94 +--- .../mapjoin/MapJoinOneLongKeyBench.java | 94 +--- .../mapjoin/MapJoinOneStringKeyBench.java | 94 +--- pom.xml | 6 +- .../hadoop/hive/ql/log/LogDivertAppender.java | 25 +++--- .../hive/ql/log/LogDivertAppenderForTest.java | 18 ++-- 6 files changed, 27 insertions(+), 304 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/5f13f286/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java -- diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java index f183bb5..7a5b721 100644 --- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java +++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/mapjoin/MapJoinMultiKeyBench.java @@ -18,107 +18,15 @@ package org.apache.hive.benchmark.vectorization.mapjoin; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import org.apache.commons.lang.ArrayUtils; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe; -import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorBatchDebug; -import org.apache.hadoop.hive.ql.exec.vector.VectorColumnOutputMapping; -import org.apache.hadoop.hive.ql.exec.vector.VectorColumnSourceMapping; -import org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow; -import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOuterFilteredOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource; -import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; -import org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator; -import org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator.GenerateType; -import org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator.GenerateType.GenerateCategory; -import org.apache.hadoop.hive.ql.exec.vector.expressions.ColAndCol; -import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestConfig.MapJoinTestImplementation; -import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestData; -import org.apache.hadoop.hive.ql.exec.vector.mapjoin.MapJoinTestDescription; -import org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.VectorMapJoinFastMultiKeyHashMap; -import org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.VectorMapJoinFastTableContainer; -import org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.VerifyFastRow; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.JoinCondDesc; -import org.apache.hadoop.hive.ql.plan.JoinDesc; -import org.apache
hive git commit: HIVE-17357: Plugin jars are not properly added for LocalHiveSparkClient (Aihua Xu, reviewed by Xuefu Zhang)
Repository: hive Updated Branches: refs/heads/master d012f8653 -> 78efa7a69 HIVE-17357: Plugin jars are not properly added for LocalHiveSparkClient (Aihua Xu, reviewed by Xuefu Zhang) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/78efa7a6 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/78efa7a6 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/78efa7a6 Branch: refs/heads/master Commit: 78efa7a69b965f39a6f0883863052acbb0bc824c Parents: d012f86 Author: Aihua Xu Authored: Fri Aug 18 10:10:42 2017 -0700 Committer: Aihua Xu Committed: Mon Aug 21 10:25:26 2017 -0700 -- .../org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/78efa7a6/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java index beeafd0..72f2f91 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java @@ -178,7 +178,7 @@ public class LocalHiveSparkClient implements HiveSparkClient { for (BaseWork work : sparkWork.getAllWork()) { work.configureJobConf(jobConf); } -addJars(conf.get(MR_JAR_PROPERTY)); +addJars(jobConf.get(MR_JAR_PROPERTY)); // add added files String addedFiles = Utilities.getResourceFiles(conf, SessionState.ResourceType.FILE);
[hive] Git Push Summary
Repository: hive Updated Branches: refs/heads/master_backport [deleted] f756bf9c5
hive git commit: HIVE-17336: Missing class 'org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat' from Hive on Spark when inserting into hbase based table (Aihua Xu, reviewed by Vihang Karajgaonkar)
Repository: hive Updated Branches: refs/heads/master 002626d5e -> f756bf9c5 HIVE-17336: Missing class 'org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat' from Hive on Spark when inserting into hbase based table (Aihua Xu, reviewed by Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f756bf9c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f756bf9c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f756bf9c Branch: refs/heads/master Commit: f756bf9c59b2bbb3e9de8d73f2fb203a0c388550 Parents: 002626d Author: Aihua Xu Authored: Wed Aug 16 15:53:55 2017 -0700 Committer: Aihua Xu Committed: Thu Aug 17 08:33:15 2017 -0700 -- .../apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/f756bf9c/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java index 7091cea..78d5ff2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java @@ -238,7 +238,8 @@ public class RemoteHiveSparkClient implements HiveSparkClient { for (BaseWork work : sparkWork.getAllWork()) { work.configureJobConf(jobConf); } -addJars(conf.get(MR_JAR_PROPERTY)); +addJars(jobConf.get(MR_JAR_PROPERTY)); + // remove the location of container tokens conf.unset(MR_CREDENTIALS_LOCATION_PROPERTY); // add added files
hive git commit: HIVE-17336: Missing class 'org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat' from Hive on Spark when inserting into hbase based table (Aihua Xu, reviewed by Vihang Karajgaonkar)
Repository: hive Updated Branches: refs/heads/master_backport [created] f756bf9c5 HIVE-17336: Missing class 'org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat' from Hive on Spark when inserting into hbase based table (Aihua Xu, reviewed by Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f756bf9c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f756bf9c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f756bf9c Branch: refs/heads/master_backport Commit: f756bf9c59b2bbb3e9de8d73f2fb203a0c388550 Parents: 002626d Author: Aihua Xu Authored: Wed Aug 16 15:53:55 2017 -0700 Committer: Aihua Xu Committed: Thu Aug 17 08:33:15 2017 -0700 -- .../apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/f756bf9c/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java index 7091cea..78d5ff2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java @@ -238,7 +238,8 @@ public class RemoteHiveSparkClient implements HiveSparkClient { for (BaseWork work : sparkWork.getAllWork()) { work.configureJobConf(jobConf); } -addJars(conf.get(MR_JAR_PROPERTY)); +addJars(jobConf.get(MR_JAR_PROPERTY)); + // remove the location of container tokens conf.unset(MR_CREDENTIALS_LOCATION_PROPERTY); // add added files
hive git commit: HIVE-17272: when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE (Aihua Xu, reviewed by Vihang Karajgaonkar)
Repository: hive Updated Branches: refs/heads/master 22659ca43 -> e9347ab97 HIVE-17272: when hive.vectorized.execution.enabled is true, query on empty partitioned table fails with NPE (Aihua Xu, reviewed by Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e9347ab9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e9347ab9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e9347ab9 Branch: refs/heads/master Commit: e9347ab9778608c415174e40ed63bcf36054b66d Parents: 22659ca Author: Aihua Xu Authored: Mon Aug 14 13:15:16 2017 -0700 Committer: Aihua Xu Committed: Wed Aug 16 10:16:51 2017 -0700 -- .../hive/ql/exec/vector/VectorMapOperator.java | 7 +++ .../queries/clientpositive/vectorized_parquet.q | 3 +++ .../clientpositive/llap/vectorized_parquet.q.out| 16 3 files changed, 26 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/e9347ab9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java index 1ac8914..e8c73a9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java @@ -385,6 +385,9 @@ public class VectorMapOperator extends AbstractMapOperator { throws SerDeException, Exception { VectorPartitionDesc vectorPartDesc = partDesc.getVectorPartitionDesc(); +if (vectorPartDesc == null) { + return null; +} VectorPartitionContext vectorPartitionContext; VectorMapOperatorReadType vectorMapOperatorReadType = vectorPartDesc.getVectorMapOperatorReadType(); @@ -631,6 +634,10 @@ public class VectorMapOperator extends AbstractMapOperator { private void setupPartitionContextVars(String nominalPath) throws HiveException { currentVectorPartContext = fileToPartitionContextMap.get(nominalPath); +if (currentVectorPartContext == null) { + return; +} + PartitionDesc partDesc = currentVectorPartContext.getPartDesc(); VectorPartitionDesc vectorPartDesc = partDesc.getVectorPartitionDesc(); currentReadType = vectorPartDesc.getVectorMapOperatorReadType(); http://git-wip-us.apache.org/repos/asf/hive/blob/e9347ab9/ql/src/test/queries/clientpositive/vectorized_parquet.q -- diff --git a/ql/src/test/queries/clientpositive/vectorized_parquet.q b/ql/src/test/queries/clientpositive/vectorized_parquet.q index e6ebdaa..db02ec0 100644 --- a/ql/src/test/queries/clientpositive/vectorized_parquet.q +++ b/ql/src/test/queries/clientpositive/vectorized_parquet.q @@ -46,3 +46,6 @@ select ctinyint, stddev_pop(cdouble) from alltypes_parquet group by ctinyint; + +CREATE TABLE empty_parquet(x int) PARTITIONED BY (y int) stored as parquet; +select * from empty_parquet t1 join empty_parquet t2 where t1.x=t2.x; http://git-wip-us.apache.org/repos/asf/hive/blob/e9347ab9/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out b/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out index e904286..8a84d3d 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out @@ -348,3 +348,19 @@ NULL 1073418988 -16379 3115NULL 305051.4870777435 55 626923679 -15887 21 55.09826.38569192808 61 626923679 -15894 29 61.08785.714950987198 62 626923679 -14307 17 62.09491.752726667326 +PREHOOK: query: CREATE TABLE empty_parquet(x int) PARTITIONED BY (y int) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@empty_parquet +POSTHOOK: query: CREATE TABLE empty_parquet(x int) PARTITIONED BY (y int) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@empty_parquet +PREHOOK: query: select * from empty_parquet t1 join empty_parquet t2 where t1.x=t2.x +PREHOOK: type: QUERY +PREHOOK: Input: default@empty_parquet + A masked pattern was here +POSTHOOK: query: select * from empty_parquet t1 join empty_parquet t2 where t1.x=t2.x +POSTHOOK: type: QUERY +POSTHOOK: Input: default@empty_parquet + A masked pattern was here
hive git commit: HIVE-16873: Remove Thread Cache From Logging (BELUGA BEHR reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 4f042cc46 -> 06d9a6bf1 HIVE-16873: Remove Thread Cache From Logging (BELUGA BEHR reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/06d9a6bf Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/06d9a6bf Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/06d9a6bf Branch: refs/heads/master Commit: 06d9a6bf142534db772f457c4d4a0c2a0c9cfe2e Parents: 4f042cc Author: Aihua Xu Authored: Mon Aug 14 13:37:57 2017 -0700 Committer: Aihua Xu Committed: Mon Aug 14 13:37:57 2017 -0700 -- .../hadoop/hive/metastore/HiveMetaStore.java | 18 ++ 1 file changed, 2 insertions(+), 16 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/06d9a6bf/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java -- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 5b353c4..5405058 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -295,20 +295,9 @@ public class HiveMetaStore extends ThriftHiveMetastore { private static ExecutorService threadPool; -public static final String AUDIT_FORMAT = -"ugi=%s\t" + // ugi -"ip=%s\t" + // remote IP -"cmd=%s\t"; // command public static final Logger auditLog = LoggerFactory.getLogger( HiveMetaStore.class.getName() + ".audit"); -private static final ThreadLocal auditFormatter = -new ThreadLocal() { - @Override - protected Formatter initialValue() { -return new Formatter(new StringBuilder(AUDIT_FORMAT.length() * 4)); - } -}; - + private static final void logAuditEvent(String cmd) { if (cmd == null) { return; @@ -320,16 +309,13 @@ public class HiveMetaStore extends ThriftHiveMetastore { } catch (Exception ex) { throw new RuntimeException(ex); } - final Formatter fmt = auditFormatter.get(); - ((StringBuilder) fmt.out()).setLength(0); String address = getIPAddress(); if (address == null) { address = "unknown-ip-addr"; } - auditLog.info(fmt.format(AUDIT_FORMAT, ugi.getUserName(), - address, cmd).toString()); + auditLog.info("ugi={}ip={} cmd={} ", ugi.getUserName(), address, cmd); } private static String getIPAddress() {
hive git commit: HIVE-17155: findConfFile() in HiveConf.java has some issues with the conf path (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master 3c7fb2a97 -> 4af249581 HIVE-17155: findConfFile() in HiveConf.java has some issues with the conf path (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4af24958 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4af24958 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4af24958 Branch: refs/heads/master Commit: 4af249581a4a25bd92c9c2dd9a09e590a0cf2831 Parents: 3c7fb2a Author: Aihua Xu Authored: Tue Jul 25 08:14:40 2017 -0700 Committer: Aihua Xu Committed: Tue Jul 25 08:14:40 2017 -0700 -- .../src/java/org/apache/hadoop/hive/conf/HiveConf.java | 12 +--- 1 file changed, 9 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/4af24958/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java -- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 3cf76d0..f5e5974 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -161,19 +161,25 @@ public class HiveConf extends Configuration { result = checkConfigFile(new File(confPath, name)); if (result == null) { String homePath = System.getenv("HIVE_HOME"); -String nameInConf = "conf" + File.pathSeparator + name; +String nameInConf = "conf" + File.separator + name; result = checkConfigFile(new File(homePath, nameInConf)); if (result == null) { URI jarUri = null; try { -jarUri = HiveConf.class.getProtectionDomain().getCodeSource().getLocation().toURI(); +// Handle both file:// and jar:!{entry} in the case of shaded hive libs +URL sourceUrl = HiveConf.class.getProtectionDomain().getCodeSource().getLocation(); +jarUri = sourceUrl.getProtocol().equalsIgnoreCase("jar") ? new URI(sourceUrl.getPath()) : sourceUrl.toURI(); } catch (Throwable e) { if (l4j.isInfoEnabled()) { l4j.info("Cannot get jar URI", e); } System.err.println("Cannot get jar URI: " + e.getMessage()); } - result = checkConfigFile(new File(new File(jarUri).getParentFile(), nameInConf)); + // From the jar file, the parent is /lib folder + File parent = new File(jarUri).getParentFile(); + if (parent != null) { +result = checkConfigFile(new File(parent.getParentFile(), nameInConf)); + } } } }
hive git commit: HIVE-17149: Hdfs directory is not cleared if partition creation failed on HMS (Barna Zsombor Klara, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master a282e5d94 -> 549ac12dd HIVE-17149: Hdfs directory is not cleared if partition creation failed on HMS (Barna Zsombor Klara, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/549ac12d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/549ac12d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/549ac12d Branch: refs/heads/master Commit: 549ac12dd3d41215aeb31d6d247a1c86f2ed85de Parents: a282e5d Author: Aihua Xu Authored: Mon Jul 24 16:21:07 2017 -0700 Committer: Aihua Xu Committed: Mon Jul 24 16:21:07 2017 -0700 -- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 10 ++ 1 file changed, 10 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/549ac12d/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 02cea7c..d661f10 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1781,6 +1781,16 @@ public class Hive { // In that case, we want to retry with alterPartition. LOG.debug("Caught AlreadyExistsException, trying to alter partition instead"); setStatsPropAndAlterPartition(hasFollowingStatsTask, tbl, newTPart); +} catch (Exception e) { + try { +final FileSystem newPathFileSystem = newPartPath.getFileSystem(this.getConf()); +boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); +final FileStatus status = newPathFileSystem.getFileStatus(newPartPath); +Hive.trashFiles(newPathFileSystem, new FileStatus[] {status}, this.getConf(), isAutoPurge); + } catch (IOException io) { +LOG.error("Could not delete partition directory contents after failed partition creation: ", io); + } + throw e; } } else { setStatsPropAndAlterPartition(hasFollowingStatsTask, tbl, newTPart);
hive git commit: HIVE-16911: Upgrade groovy version to 2.4.11 (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master 7f5460d8e -> 45a1ec87e HIVE-16911: Upgrade groovy version to 2.4.11 (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/45a1ec87 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/45a1ec87 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/45a1ec87 Branch: refs/heads/master Commit: 45a1ec87e84fc3803337232c981726b9ae7bae64 Parents: 7f5460d Author: Aihua Xu Authored: Mon Jul 10 14:51:54 2017 -0700 Committer: Aihua Xu Committed: Mon Jul 10 14:51:54 2017 -0700 -- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/45a1ec87/pom.xml -- diff --git a/pom.xml b/pom.xml index f9fae59..04cbf11 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ 0.1.2 0.10.0 14.0.1 -2.4.4 +2.4.11 1.3.166 2.8.0 ${basedir}/${hive.path.to.root}/testutils/hadoop
hive git commit: HIVE-17048: Pass HiveOperation info to HiveSemanticAnalyzerHook through HiveSemanticAnalyzerHookContext (Aihua Xu, reviewed by Mohit Sabharwal)
Repository: hive Updated Branches: refs/heads/master 453f44cdb -> 6b87af747 HIVE-17048: Pass HiveOperation info to HiveSemanticAnalyzerHook through HiveSemanticAnalyzerHookContext (Aihua Xu, reviewed by Mohit Sabharwal) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6b87af74 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6b87af74 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6b87af74 Branch: refs/heads/master Commit: 6b87af7477219a3b62acb4b8ff4e614d45816d68 Parents: 453f44c Author: Aihua Xu Authored: Wed Jul 5 17:38:36 2017 -0700 Committer: Aihua Xu Committed: Fri Jul 7 10:46:43 2017 -0700 -- .../hive/minikdc/TestHs2HooksWithMiniKdc.java | 28 +++- .../apache/hadoop/hive/hooks/TestHs2Hooks.java | 27 --- .../java/org/apache/hadoop/hive/ql/Driver.java | 1 + .../parse/HiveSemanticAnalyzerHookContext.java | 5 .../HiveSemanticAnalyzerHookContextImpl.java| 12 + 5 files changed, 50 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/6b87af74/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java -- diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java index 4fabe47..1cd0ee8 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java @@ -24,7 +24,7 @@ import java.sql.Statement; import java.util.HashMap; import java.util.Map; -import junit.framework.Assert; +import org.junit.Assert; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -114,16 +114,16 @@ public class TestHs2HooksWithMiniKdc { throw error; } -Assert.assertNotNull(PostExecHook.ipAddress, "ipaddress is null"); -Assert.assertNotNull(PostExecHook.userName, "userName is null"); -Assert.assertNotNull(PostExecHook.operation , "operation is null"); +Assert.assertNotNull("ipaddress is null", PostExecHook.ipAddress); +Assert.assertNotNull("userName is null", PostExecHook.userName); +Assert.assertNotNull("operation is null", PostExecHook.operation); Assert.assertEquals(MiniHiveKdc.HIVE_TEST_USER_1, PostExecHook.userName); Assert.assertTrue(PostExecHook.ipAddress, PostExecHook.ipAddress.contains("127.0.0.1")); Assert.assertEquals("SHOWTABLES", PostExecHook.operation); -Assert.assertNotNull(PreExecHook.ipAddress, "ipaddress is null"); -Assert.assertNotNull(PreExecHook.userName, "userName is null"); -Assert.assertNotNull(PreExecHook.operation , "operation is null"); +Assert.assertNotNull("ipaddress is null", PreExecHook.ipAddress); +Assert.assertNotNull("userName is null", PreExecHook.userName); +Assert.assertNotNull("operation is null", PreExecHook.operation); Assert.assertEquals(MiniHiveKdc.HIVE_TEST_USER_1, PreExecHook.userName); Assert.assertTrue(PreExecHook.ipAddress, PreExecHook.ipAddress.contains("127.0.0.1")); Assert.assertEquals("SHOWTABLES", PreExecHook.operation); @@ -137,12 +137,14 @@ public class TestHs2HooksWithMiniKdc { throw error; } -Assert.assertNotNull(SemanticAnalysisHook.ipAddress, -"semantic hook context ipaddress is null"); -Assert.assertNotNull(SemanticAnalysisHook.userName, -"semantic hook context userName is null"); -Assert.assertNotNull(SemanticAnalysisHook.command , -"semantic hook context command is null"); +Assert.assertNotNull("semantic hook context ipaddress is null", +SemanticAnalysisHook.ipAddress); +Assert.assertNotNull("semantic hook context userName is null", +SemanticAnalysisHook.userName); +Assert.assertNotNull("semantic hook context command is null", +SemanticAnalysisHook.command); +Assert.assertNotNull("semantic hook context commandType is null", +SemanticAnalysisHook.commandType); Assert.assertTrue(SemanticAnalysisHook.ipAddress, SemanticAnalysisHook.ipAddress.contains("127.0.0.1")); Assert.assertEquals("show tables", SemanticAnalysisHook.command); http://git-wip-us.apache.org/repos/asf/hive/blob/6b87af74/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java index 6f3e779..dad516c 100644 --- a/itests/hive-
hive git commit: HIVE-16902: investigate "failed to remove operation log" errors (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master f6be1a3c0 -> cb6bf8871 HIVE-16902: investigate "failed to remove operation log" errors (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cb6bf887 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cb6bf887 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cb6bf887 Branch: refs/heads/master Commit: cb6bf8871ebaed2756d734368f644fbd4f6ad5d1 Parents: f6be1a3 Author: Aihua Xu Authored: Wed Jun 14 17:38:32 2017 -0400 Committer: Aihua Xu Committed: Fri Jun 16 10:39:02 2017 -0400 -- .../org/apache/hadoop/hive/ql/session/OperationLog.java | 6 +++--- .../org/apache/hive/service/cli/operation/Operation.java| 9 + 2 files changed, 4 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/cb6bf887/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java index ded4b18..1d1fc4e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java @@ -51,7 +51,7 @@ public class OperationLog { NONE, EXECUTION, PERFORMANCE, VERBOSE, UNKNOWN } - public OperationLog(String name, File file, HiveConf hiveConf) throws FileNotFoundException { + public OperationLog(String name, File file, HiveConf hiveConf) { operationName = name; logFile = new LogFile(file); @@ -133,7 +133,7 @@ public class OperationLog { private BufferedReader in; private volatile boolean isRemoved; -LogFile(File file) throws FileNotFoundException { +LogFile(File file) { this.file = file; isRemoved = false; } @@ -157,7 +157,7 @@ public class OperationLog { if (in != null) { in.close(); } -if (!isRemoved && removeLog) { +if (!isRemoved && removeLog && file.exists()) { FileUtils.forceDelete(file); isRemoved = true; } http://git-wip-us.apache.org/repos/asf/hive/blob/cb6bf887/service/src/java/org/apache/hive/service/cli/operation/Operation.java -- diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java index 4e78551..8d453d5 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java @@ -211,14 +211,7 @@ public abstract class Operation { isOperationLogEnabled = true; // create OperationLog object with above log file - try { -operationLog = new OperationLog(opHandle.toString(), operationLogFile, parentSession.getHiveConf()); - } catch (FileNotFoundException e) { -LOG.warn("Unable to instantiate OperationLog object for operation: " + -opHandle, e); -isOperationLogEnabled = false; -return; - } + operationLog = new OperationLog(opHandle.toString(), operationLogFile, parentSession.getHiveConf()); } }
hive git commit: HIVE-16846: TestJdbcWithMiniHS2#testHttpHeaderSize test case is not testing in HTTP mode (Aihua Xu, reviewed by YongZhi Chen)
Repository: hive Updated Branches: refs/heads/master 4a8eaa571 -> 44653b57c HIVE-16846: TestJdbcWithMiniHS2#testHttpHeaderSize test case is not testing in HTTP mode (Aihua Xu, reviewed by YongZhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/44653b57 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/44653b57 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/44653b57 Branch: refs/heads/master Commit: 44653b57c2576f76359f2cf2ff2ef883b087959b Parents: 4a8eaa5 Author: Aihua Xu Authored: Wed Jun 7 15:26:46 2017 -0400 Committer: Aihua Xu Committed: Fri Jun 9 10:51:14 2017 -0400 -- .../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 36 +--- 1 file changed, 24 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/44653b57/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index fc2cb08..4a9af80 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -195,9 +195,17 @@ public class TestJdbcWithMiniHS2 { } private static void startMiniHS2(HiveConf conf) throws Exception { +startMiniHS2(conf, false); + } + + private static void startMiniHS2(HiveConf conf, boolean httpMode) throws Exception { conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false); -miniHS2 = new MiniHS2.Builder().withConf(conf).cleanupLocalDirOnStartup(false).build(); +MiniHS2.Builder builder = new MiniHS2.Builder().withConf(conf).cleanupLocalDirOnStartup(false); +if (httpMode) { + builder = builder.withHTTPTransport(); +} +miniHS2 = builder.build(); Map confOverlay = new HashMap(); miniHS2.start(confOverlay); } @@ -923,10 +931,9 @@ public class TestJdbcWithMiniHS2 { // Stop HiveServer2 stopMiniHS2(); HiveConf conf = new HiveConf(); -conf.set("hive.server2.transport.mode", "http"); -conf.setInt("hive.server2.thrift.http.request.header.size", 1024); -conf.setInt("hive.server2.thrift.http.response.header.size", 1024); -startMiniHS2(conf); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_REQUEST_HEADER_SIZE, 1024); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_RESPONSE_HEADER_SIZE, 1024); +startMiniHS2(conf, true); // Username is added to the request header String userName = StringUtils.leftPad("*", 100); @@ -945,25 +952,31 @@ public class TestJdbcWithMiniHS2 { // This should fail with given HTTP response code 413 in error message, since header is more // than the configured the header size userName = StringUtils.leftPad("*", 2000); +Exception headerException = null; try { + conn = null; conn = getConnection(miniHS2.getJdbcURL(testDbName), userName, "password"); } catch (Exception e) { - assertTrue("Header exception thrown", e != null); - assertTrue(e.getMessage().contains("HTTP Response code: 413")); + headerException = e; } finally { if (conn != null) { conn.close(); } + + assertTrue("Header exception should be thrown", headerException != null); + assertTrue("Incorrect HTTP Response:" + headerException.getMessage(), + headerException.getMessage().contains("HTTP Response code: 413")); } // Stop HiveServer2 to increase header size stopMiniHS2(); -conf.setInt("hive.server2.thrift.http.request.header.size", 3000); -conf.setInt("hive.server2.thrift.http.response.header.size", 3000); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_REQUEST_HEADER_SIZE, 3000); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_RESPONSE_HEADER_SIZE, 3000); startMiniHS2(conf); // This should now go fine, since we increased the configured header size try { + conn = null; conn = getConnection(miniHS2.getJdbcURL(testDbName), userName, "password"); } catch (Exception e) { fail("Not expecting exception: " + e); @@ -986,10 +999,9 @@ public class TestJdbcWithMiniHS2 { // Stop HiveServer2 stopMiniHS2(); HiveConf conf = new HiveConf(); -conf.set("hive.server2.transport.mode", "http"); // Set server's idle timeout to a very low value -conf.set("hive.server2.thrift.http.max.idle.time", "5"); -startMiniHS2(conf); +conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIM
hive git commit: HIVE-16769: Possible hive service startup due to the existing file /tmp/stderr (Aihua Xu, reviewed by Naveen Gangam)
Repository: hive Updated Branches: refs/heads/master 3f5e65288 -> 9f976 HIVE-16769: Possible hive service startup due to the existing file /tmp/stderr (Aihua Xu, reviewed by Naveen Gangam) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f976fff Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f976fff Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f976fff Branch: refs/heads/master Commit: 9f976343a891c9d3e8d5fa0e70b2c622ff1a Parents: 3f5e652 Author: Aihua Xu Authored: Wed May 31 17:08:29 2017 -0400 Committer: Aihua Xu Committed: Fri Jun 2 09:21:39 2017 -0400 -- bin/ext/version.sh | 2 +- bin/hive | 20 ++-- 2 files changed, 3 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/9f976fff/bin/ext/version.sh -- diff --git a/bin/ext/version.sh b/bin/ext/version.sh index b6a237b..92fee3d 100644 --- a/bin/ext/version.sh +++ b/bin/ext/version.sh @@ -27,7 +27,7 @@ version () { # hadoop 20 or newer - skip the aux_jars option and hiveconf CLASS=org.apache.hive.common.util.HiveVersionInfo - exec $HADOOP jar $JAR $CLASS 2>> ${STDERR} + exec $HADOOP jar $JAR $CLASS 2>&2 } version_help () { http://git-wip-us.apache.org/repos/asf/hive/blob/9f976fff/bin/hive -- diff --git a/bin/hive b/bin/hive index a7671c3..4a908e1 100755 --- a/bin/hive +++ b/bin/hive @@ -25,8 +25,6 @@ bin=`cd "$bin"; pwd` . "$bin"/hive-config.sh -TMP_USER_DIR="/tmp/${USER}" -STDERR="${TMP_USER_DIR}/stderr" SERVICE="" HELP="" SKIP_HBASECP=false @@ -227,24 +225,10 @@ if [ ! -f ${HADOOP} ]; then exit 4; fi -if [ ! -d ${TMP_USER_DIR} ]; then - mkdir -p ${TMP_USER_DIR} 2> /dev/null - if [ $? -ne 0 ]; then -STDERR="/dev/tty" - fi -fi - -if [ "${STDERR}" != "/dev/null" ] && [ ! -f ${STDERR} ]; then - touch ${STDERR} 2> /dev/null - if [ $? -ne 0 ]; then -STDERR="/dev/tty" - fi -fi - if [ "$SKIP_HADOOPVERSION" = false ]; then # Make sure we're using a compatible version of Hadoop if [ "x$HADOOP_VERSION" == "x" ]; then - HADOOP_VERSION=$($HADOOP version 2>> ${STDERR} | awk -F"\t" '/Hadoop/ {print $0}' | cut -d' ' -f 2); + HADOOP_VERSION=$($HADOOP version 2>&2 | awk -F"\t" '/Hadoop/ {print $0}' | cut -d' ' -f 2); fi # Save the regex to a var to workaround quoting incompatabilities @@ -295,7 +279,7 @@ if [ "$SKIP_HBASECP" = false ]; then if [[ -n $HBASE_BIN ]] ; then # exclude ZK, PB, and Guava (See HIVE-2055) # depends on HBASE-8438 (hbase-0.94.14+, hbase-0.96.1+) for `hbase mapredcp` command -for x in $($HBASE_BIN mapredcp 2>> ${STDERR} | tr ':' '\n') ; do +for x in $($HBASE_BIN mapredcp 2>&2 | tr ':' '\n') ; do if [[ $x == *zookeeper* || $x == *protobuf-java* || $x == *guava* ]] ; then continue fi
hive git commit: HIVE-16625: Extra '\0' characters in the output, when SeparatedValuesOutputFormat is used and the quoting is disabled (Peter Vary, reviewed by Zoltan Haindrich and Aihua Xu)
Repository: hive Updated Branches: refs/heads/master f5b225021 -> 396cba458 HIVE-16625: Extra '\0' characters in the output, when SeparatedValuesOutputFormat is used and the quoting is disabled (Peter Vary, reviewed by Zoltan Haindrich and Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/396cba45 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/396cba45 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/396cba45 Branch: refs/heads/master Commit: 396cba45822ab9e1519504cdde82df1d89b60c8f Parents: f5b2250 Author: Aihua Xu Authored: Wed May 31 13:00:04 2017 -0400 Committer: Aihua Xu Committed: Wed May 31 13:01:47 2017 -0400 -- .../beeline/SeparatedValuesOutputFormat.java| 11 +- .../test/resources/testconfiguration.properties |1 + .../java/org/apache/hive/beeline/QFile.java |1 + .../insert_overwrite_local_directory_1.q.out| 1769 ++ 4 files changed, 1780 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/396cba45/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java -- diff --git a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java index 66d9fd0..9c24a20 100644 --- a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java +++ b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java @@ -26,6 +26,8 @@ import java.io.IOException; import java.io.StringWriter; import org.apache.hadoop.io.IOUtils; +import org.supercsv.encoder.CsvEncoder; +import org.supercsv.encoder.SelectiveCsvEncoder; import org.supercsv.io.CsvListWriter; import org.supercsv.prefs.CsvPreference; @@ -40,10 +42,15 @@ class SeparatedValuesOutputFormat implements OutputFormat { SeparatedValuesOutputFormat(BeeLine beeLine, char separator) { this.beeLine = beeLine; -unquotedCsvPreference = new CsvPreference.Builder('\0', separator, "").build(); +unquotedCsvPreference = getUnquotedCsvPreference(separator); quotedCsvPreference = new CsvPreference.Builder('"', separator, "").build(); } + private static CsvPreference getUnquotedCsvPreference(char delimiter) { +CsvEncoder noEncoder = new SelectiveCsvEncoder(); +return new CsvPreference.Builder('\0', delimiter, "").useEncoder(noEncoder).build(); + } + private void updateCsvPreference() { if (beeLine.getOpts().getOutputFormat().equals("dsv")) { // check whether delimiter changed by user @@ -54,7 +61,7 @@ class SeparatedValuesOutputFormat implements OutputFormat { // "" is passed as the end of line symbol in following function, as // beeline itself adds newline if (isQuotingDisabled()) { - unquotedCsvPreference = new CsvPreference.Builder('\0', newDel, "").build(); + unquotedCsvPreference = getUnquotedCsvPreference(newDel); } else { quotedCsvPreference = new CsvPreference.Builder('"', newDel, "").build(); } http://git-wip-us.apache.org/repos/asf/hive/blob/396cba45/itests/src/test/resources/testconfiguration.properties -- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 47a13c9..f4a53df 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -759,6 +759,7 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\ beeline.positive.include=create_merge_compressed.q,\ drop_with_concurrency.q,\ escape_comments.q,\ + insert_overwrite_local_directory_1.q,\ mapjoin2.q,\ materialized_view_create_rewrite.q,\ smb_mapjoin_1.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/396cba45/itests/util/src/main/java/org/apache/hive/beeline/QFile.java -- diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java index 0d00a9b..e70ac38 100644 --- a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java +++ b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java @@ -304,6 +304,7 @@ public final class QFile { .addFilter(".*/tmp/.*\n", MASK_PATTERN) .addFilter(".*file:.*\n", MASK_PATTERN) .addFilter(".*file\\..*\n", MASK_PATTERN) +.addFilter(".*Output:.*/data/files/.*\n", MASK_PATTERN) .addFilter(".*CreateTime.*\n", MASK_PATTERN) .addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN) .addFilter(".*lastUpdateT
hive git commit: HIVE-16468: BeeLineDriver should be able to run tests against an externally created cluster (Peter Vary, reviewed by Vihang Karajgaonkar and Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 3be1eedb1 -> 4045ca399 HIVE-16468: BeeLineDriver should be able to run tests against an externally created cluster (Peter Vary, reviewed by Vihang Karajgaonkar and Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4045ca39 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4045ca39 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4045ca39 Branch: refs/heads/master Commit: 4045ca399228a8cac8005131cacee95568e6999a Parents: 3be1eed Author: Aihua Xu Authored: Fri May 19 11:42:28 2017 -0400 Committer: Aihua Xu Committed: Fri May 19 11:47:20 2017 -0400 -- .../hive/cli/control/CoreBeeLineDriver.java | 58 1 file changed, 34 insertions(+), 24 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/4045ca39/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java -- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java index 2be83ca..9dfc253 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java @@ -20,7 +20,9 @@ package org.apache.hadoop.hive.cli.control; import static org.junit.Assert.fail; import com.google.common.base.Strings; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.hooks.PreExecutePrinter; import org.apache.hive.beeline.ConvertedOutputFile.Converter; @@ -52,19 +54,40 @@ public class CoreBeeLineDriver extends CliAdapter { private QFileClientBuilder clientBuilder; private QFileBuilder fileBuilder; -// private static QTestUtil.QTestSetup miniZKCluster = null; - public CoreBeeLineDriver(AbstractCliConfig testCliConfig) { super(testCliConfig); queryDirectory = new File(testCliConfig.getQueryDirectory()); logDirectory = new File(testCliConfig.getLogDir()); resultsDirectory = new File(testCliConfig.getResultsDir()); -testDataDirectory = new File(hiveRootDirectory, "data" + File.separator + "files"); +String testDataDirectoryName = System.getProperty("test.data.dir"); +if (testDataDirectoryName == null) { + testDataDirectory = new File(hiveRootDirectory, "data" + File.separator + "files"); +} else { + testDataDirectory = new File(testDataDirectoryName); +} testScriptDirectory = new File(hiveRootDirectory, "data" + File.separator + "scripts"); initScript = new File(testScriptDirectory, testCliConfig.getInitScript()); cleanupScript = new File(testScriptDirectory, testCliConfig.getCleanupScript()); } + private static MiniHS2 createMiniServer() throws Exception { +HiveConf hiveConf = new HiveConf(); +// We do not need Zookeeper at the moment +hiveConf.set(HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname, +"org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); + +MiniHS2 miniHS2 = new MiniHS2.Builder() +.withConf(hiveConf) +.cleanupLocalDirOnStartup(true) +.build(); + +miniHS2.start(new HashMap()); + +System.err.println(HiveConfUtil.dumpConfig(miniHS2.getHiveConf())); + +return miniHS2; + } + @Override @BeforeClass public void beforeClass() throws Exception { @@ -77,27 +100,17 @@ public class CoreBeeLineDriver extends CliAdapter { rewriteSourceTables = false; } -HiveConf hiveConf = new HiveConf(); -// We do not need Zookeeper at the moment -hiveConf.set(HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname, -"org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); - -// But if we need later we can enable it with this, or create one ourself -//miniZKCluster = new QTestUtil.QTestSetup(); -//miniZKCluster.preTest(hiveConf); - -hiveConf.logVars(System.err); -System.err.flush(); - -miniHS2 = new MiniHS2.Builder().withConf(hiveConf).cleanupLocalDirOnStartup(true).build(); - -miniHS2.start(new HashMap()); +String beeLineUrl = System.getProperty("test.beeline.url"); +if (StringUtils.isEmpty(beeLineUrl)) { + miniHS2 = createMiniServer(); + beeLineUrl = miniHS2.getJdbcURL(); +} clientBuilder = new QFileClientBuilder() .setJdbcDriver("org.apache.hive.jdbc.HiveDriver") -.setJdbcUrl(miniHS2.getJdbcURL()) -.setUsername("user") -.setPassword("password"); +.setJdbcUrl(beeLineUrl) +.setUserna
hive git commit: HIVE-16324: Truncate table should not work when EXTERNAL property of table is true (Vihang Karajgaonkar via Aihua Xu, reviewed by Thejas M Nair)
Repository: hive Updated Branches: refs/heads/branch-2 446d3b86f -> d7a45b756 HIVE-16324: Truncate table should not work when EXTERNAL property of table is true (Vihang Karajgaonkar via Aihua Xu, reviewed by Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d7a45b75 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d7a45b75 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d7a45b75 Branch: refs/heads/branch-2 Commit: d7a45b75623d1df9d212da5342cb95fdd55577e6 Parents: 446d3b8 Author: Aihua Xu Authored: Mon May 15 16:08:50 2017 -0400 Committer: Aihua Xu Committed: Thu May 18 10:29:43 2017 -0400 -- .../apache/hadoop/hive/metastore/ObjectStore.java | 6 +++--- .../clientnegative/truncate_table_failure5.q | 5 + .../clientnegative/truncate_table_failure6.q | 5 + .../clientnegative/truncate_table_failure5.q.out | 17 + .../clientnegative/truncate_table_failure6.q.out | 17 + 5 files changed, 47 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/d7a45b75/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java -- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 4aa4448..3e96d4a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1147,7 +1147,7 @@ public class ObjectStore implements RawStore, Configurable { return mConstraints; } -@Override + @Override public Table getTable(String dbName, String tableName) throws MetaException { boolean commited = false; Table tbl = null; @@ -1426,7 +1426,7 @@ public class ObjectStore implements RawStore, Configurable { // for backwards compatibility with old metastore persistence if (mtbl.getViewOriginalText() != null) { tableType = TableType.VIRTUAL_VIEW.toString(); - } else if ("TRUE".equals(mtbl.getParameters().get("EXTERNAL"))) { + } else if (Boolean.parseBoolean(mtbl.getParameters().get("EXTERNAL"))) { tableType = TableType.EXTERNAL_TABLE.toString(); } else { tableType = TableType.MANAGED_TABLE.toString(); @@ -1458,7 +1458,7 @@ public class ObjectStore implements RawStore, Configurable { // If the table has property EXTERNAL set, update table type // accordingly String tableType = tbl.getTableType(); -boolean isExternal = "TRUE".equals(tbl.getParameters().get("EXTERNAL")); +boolean isExternal = Boolean.parseBoolean(tbl.getParameters().get("EXTERNAL")); if (TableType.MANAGED_TABLE.toString().equals(tableType)) { if (isExternal) { tableType = TableType.EXTERNAL_TABLE.toString(); http://git-wip-us.apache.org/repos/asf/hive/blob/d7a45b75/ql/src/test/queries/clientnegative/truncate_table_failure5.q -- diff --git a/ql/src/test/queries/clientnegative/truncate_table_failure5.q b/ql/src/test/queries/clientnegative/truncate_table_failure5.q new file mode 100644 index 000..efabb34 --- /dev/null +++ b/ql/src/test/queries/clientnegative/truncate_table_failure5.q @@ -0,0 +1,5 @@ +create table external1 (col1 int, col2 string); +alter table external1 set tblproperties ('EXTERNAL'='true'); + +-- truncate on a non-managed table should throw exception +truncate table external1; http://git-wip-us.apache.org/repos/asf/hive/blob/d7a45b75/ql/src/test/queries/clientnegative/truncate_table_failure6.q -- diff --git a/ql/src/test/queries/clientnegative/truncate_table_failure6.q b/ql/src/test/queries/clientnegative/truncate_table_failure6.q new file mode 100644 index 000..b73ec63 --- /dev/null +++ b/ql/src/test/queries/clientnegative/truncate_table_failure6.q @@ -0,0 +1,5 @@ +create table external1 (col1 int, col2 string); +alter table external1 set tblproperties ('EXTERNAL'='TRUE'); + +-- truncate on external table should throw exception. Property value of 'EXTERNAL' is not case sensitive +truncate table external1; http://git-wip-us.apache.org/repos/asf/hive/blob/d7a45b75/ql/src/test/results/clientnegative/truncate_table_failure5.q.out -- diff --git a/ql/src/test/results/clientnegative/truncate_table_failure5.q.out b/ql/src/test/results/clientnegative/truncate_table_failure5.q.out new file mode 100644 index 000..d2f625d --- /dev/null +++ b/ql/src/test/results/clientnegative/truncate_table_failure5.q.out @@ -0,0 +1,17 @@ +PREHOOK: qu
hive git commit: HIVE-16647: Improve the validation output to make the output to stderr and stdout more consistent (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive Updated Branches: refs/heads/master c19981c66 -> 2bcbd29e8 HIVE-16647: Improve the validation output to make the output to stderr and stdout more consistent (Aihua Xu, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2bcbd29e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2bcbd29e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2bcbd29e Branch: refs/heads/master Commit: 2bcbd29e8d2fbc740c762997663cc40853892add Parents: c19981c Author: Aihua Xu Authored: Thu May 11 10:11:29 2017 -0400 Committer: Aihua Xu Committed: Tue May 16 09:14:49 2017 -0400 -- .../org/apache/hive/beeline/HiveSchemaTool.java | 46 ++-- 1 file changed, 22 insertions(+), 24 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/2bcbd29e/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java -- diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index 7ab927a..86faeb0 100644 --- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -191,7 +191,7 @@ public class HiveSchemaTool { try(Statement stmt = metastoreConn.createStatement(); ResultSet res = stmt.executeQuery(versionQuery)) { if (!res.next()) { -throw new HiveMetaException("Could not find version info in metastore VERSION table"); +throw new HiveMetaException("Could not find version info in metastore VERSION table."); } String currentSchemaVersion = res.getString(1); if (checkDuplicatedVersion && res.next()) { @@ -244,7 +244,6 @@ public class HiveSchemaTool { } if (numOfInvalid > 0) { isValid = false; - System.err.println("Total number of invalid DB locations is: "+ numOfInvalid); } return isValid; } @@ -305,7 +304,6 @@ public class HiveSchemaTool { } if (numOfInvalid > 0) { isValid = false; - System.err.println("Total number of invalid TABLE locations is: "+ numOfInvalid); } return isValid; } @@ -367,7 +365,6 @@ public class HiveSchemaTool { } if (numOfInvalid > 0) { isValid = false; - System.err.println("Total number of invalid PARTITION locations is: "+ numOfInvalid); } return isValid; } @@ -384,11 +381,11 @@ public class HiveSchemaTool { } if (getDbCommandParser(dbType).needsQuotedIdentifier()) { - skewedColLoc = "select t.\"TBL_NAME\", t.\"TBL_ID\", sk.\"STRING_LIST_ID_KID\", sk.\"LOCATION\" from \"TBLS\" t, \"SDS\" s, \"SKEWED_COL_VALUE_LOC_MAP\" sk " - + "where sk.\"SD_ID\" = s.\"SD_ID\" and s.\"SD_ID\" = t.\"SD_ID\" and sk.\"STRING_LIST_ID_KID\" >= ? and sk.\"STRING_LIST_ID_KID\" <= ? "; + skewedColLoc = "select t.\"TBL_NAME\", t.\"TBL_ID\", sk.\"STRING_LIST_ID_KID\", sk.\"LOCATION\", db.\"NAME\", db.\"DB_ID\" from \"TBLS\" t, \"SDS\" s, \"DBS\" db, \"SKEWED_COL_VALUE_LOC_MAP\" sk " + + "where sk.\"SD_ID\" = s.\"SD_ID\" and s.\"SD_ID\" = t.\"SD_ID\" and t.\"DB_ID\" = db.\"DB_ID\" and sk.\"STRING_LIST_ID_KID\" >= ? and sk.\"STRING_LIST_ID_KID\" <= ? "; } else { - skewedColLoc = "select t.TBL_NAME, t.TBL_ID, sk.STRING_LIST_ID_KID, sk.LOCATION from TBLS t, SDS s, SKEWED_COL_VALUE_LOC_MAP sk " - + "where sk.SD_ID = s.SD_ID and s.SD_ID = t.SD_ID and sk.STRING_LIST_ID_KID >= ? and sk.STRING_LIST_ID_KID <= ? "; + skewedColLoc = "select t.TBL_NAME, t.TBL_ID, sk.STRING_LIST_ID_KID, sk.LOCATION, db.NAME, db.DB_ID from TBLS t, SDS s, DBS db, SKEWED_COL_VALUE_LOC_MAP sk " + + "where sk.SD_ID = s.SD_ID and s.SD_ID = t.SD_ID and t.DB_ID = db.DB_ID and sk.STRING_LIST_ID_KID >= ? and sk.STRING_LIST_ID_KID <= ? "; } long maxID = 0, minID = 0; @@ -410,7 +407,8 @@ public class HiveSchemaTool { res = pStmt.executeQuery(); while (res.next()) { String locValue = res.getString(4); - String entity = "Table " + getNameOrID(res,1,2) + + String entity = "Database " + getNameOrID(res,5,6) + + ", Table " + getNameOrID(res,1,2) + ", String list " + res.getString(3); if (!checkLocation(entity, locValue, defaultServers)) { numOfInvalid++; @@ -425,7 +423,6 @@ public class HiveSchemaTool { } if (numOfInvalid > 0) { isValid = false; - System.err.println("Total number of invalid SKEWED_COL_VALUE_LOC_MAP locations is: "+ numOfInvalid); } return isValid; } @@ -445,14 +442,14 @@ public class HiveSchemaTool { URI[] defaultServers) { boolean isValid = true; if (entityLocat
hive git commit: HIVE-16324: Truncate table should not work when EXTERNAL property of table is true (Vihang Karajgaonkar via Aihua Xu, reviewed by Thejas M Nair)
Repository: hive Updated Branches: refs/heads/master 413245ed9 -> 7d4554dd1 HIVE-16324: Truncate table should not work when EXTERNAL property of table is true (Vihang Karajgaonkar via Aihua Xu, reviewed by Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7d4554dd Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7d4554dd Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7d4554dd Branch: refs/heads/master Commit: 7d4554dd1def433e0439cdbe7dfa665b6909e706 Parents: 413245e Author: Aihua Xu Authored: Mon May 15 16:08:50 2017 -0400 Committer: Aihua Xu Committed: Mon May 15 16:08:50 2017 -0400 -- .../apache/hadoop/hive/metastore/ObjectStore.java | 6 +++--- .../hadoop/hive/metastore/cache/CachedStore.java | 2 +- .../clientnegative/truncate_table_failure5.q | 5 + .../clientnegative/truncate_table_failure6.q | 5 + .../clientnegative/truncate_table_failure5.q.out | 17 + .../clientnegative/truncate_table_failure6.q.out | 17 + 6 files changed, 48 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/7d4554dd/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java -- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index ed19f42..ee48617 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1155,7 +1155,7 @@ public class ObjectStore implements RawStore, Configurable { return mConstraints; } -@Override + @Override public Table getTable(String dbName, String tableName) throws MetaException { boolean commited = false; Table tbl = null; @@ -1434,7 +1434,7 @@ public class ObjectStore implements RawStore, Configurable { // for backwards compatibility with old metastore persistence if (mtbl.getViewOriginalText() != null) { tableType = TableType.VIRTUAL_VIEW.toString(); - } else if ("TRUE".equals(mtbl.getParameters().get("EXTERNAL"))) { + } else if (Boolean.parseBoolean(mtbl.getParameters().get("EXTERNAL"))) { tableType = TableType.EXTERNAL_TABLE.toString(); } else { tableType = TableType.MANAGED_TABLE.toString(); @@ -1466,7 +1466,7 @@ public class ObjectStore implements RawStore, Configurable { // If the table has property EXTERNAL set, update table type // accordingly String tableType = tbl.getTableType(); -boolean isExternal = "TRUE".equals(tbl.getParameters().get("EXTERNAL")); +boolean isExternal = Boolean.parseBoolean(tbl.getParameters().get("EXTERNAL")); if (TableType.MANAGED_TABLE.toString().equals(tableType)) { if (isExternal) { tableType = TableType.EXTERNAL_TABLE.toString(); http://git-wip-us.apache.org/repos/asf/hive/blob/7d4554dd/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java -- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 5a187d8..1cc838f 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -447,7 +447,7 @@ public class CachedStore implements RawStore, Configurable { // If the table has property EXTERNAL set, update table type // accordingly String tableType = tbl.getTableType(); -boolean isExternal = "TRUE".equals(tbl.getParameters().get("EXTERNAL")); +boolean isExternal = Boolean.parseBoolean(tbl.getParameters().get("EXTERNAL")); if (TableType.MANAGED_TABLE.toString().equals(tableType)) { if (isExternal) { tableType = TableType.EXTERNAL_TABLE.toString(); http://git-wip-us.apache.org/repos/asf/hive/blob/7d4554dd/ql/src/test/queries/clientnegative/truncate_table_failure5.q -- diff --git a/ql/src/test/queries/clientnegative/truncate_table_failure5.q b/ql/src/test/queries/clientnegative/truncate_table_failure5.q new file mode 100644 index 000..efabb34 --- /dev/null +++ b/ql/src/test/queries/clientnegative/truncate_table_failure5.q @@ -0,0 +1,5 @@ +create table external1 (col1 int, col2 string); +alter table external1 set tblproperties ('EXTERNAL'='true'); + +-- truncate on a non-managed table should throw exception +truncate table external1; http://git-wip-us.apache.org/repos/asf/hive/blob/7d4554dd/ql/src/test/qu
hive git commit: HIVE-16143: Improve msck repair batching (Vihang Karajgaonkar, reviewed by Sahil Takiar & Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 86f74fdd2 -> 0efb93681 HIVE-16143: Improve msck repair batching (Vihang Karajgaonkar, reviewed by Sahil Takiar & Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0efb9368 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0efb9368 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0efb9368 Branch: refs/heads/master Commit: 0efb93681099af4e4b7269c72d86fe379c99da7c Parents: 86f74fd Author: Aihua Xu Authored: Mon May 15 14:08:32 2017 -0400 Committer: Aihua Xu Committed: Mon May 15 14:08:32 2017 -0400 -- .../org/apache/hadoop/hive/conf/HiveConf.java | 13 +- .../apache/hive/common/util/RetryUtilities.java | 112 ++ .../hive/common/util/TestRetryUtilities.java| 150 .../test/queries/clientpositive/create_like.q | 3 +- .../results/clientpositive/create_like.q.out| 15 +- .../org/apache/hadoop/hive/ql/QTestUtil.java| 3 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 89 ++--- .../exec/TestMsckCreatePartitionsInBatches.java | 340 +++ .../test/queries/clientpositive/msck_repair_0.q | 10 + .../test/queries/clientpositive/msck_repair_1.q | 8 + .../test/queries/clientpositive/msck_repair_2.q | 5 + .../test/queries/clientpositive/msck_repair_3.q | 4 + .../clientpositive/msck_repair_batchsize.q | 4 + .../results/clientpositive/msck_repair_0.q.out | 38 ++- .../results/clientpositive/msck_repair_1.q.out | 28 +- .../results/clientpositive/msck_repair_2.q.out | 28 +- .../results/clientpositive/msck_repair_3.q.out | 28 +- .../clientpositive/msck_repair_batchsize.q.out | 34 +- ql/src/test/results/clientpositive/repair.q.out | 3 +- 19 files changed, 857 insertions(+), 58 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/0efb9368/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java -- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6068f0d..1c37b6e 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3369,9 +3369,16 @@ public class HiveConf extends Configuration { " others; 'ignore' will skip the validation (legacy behavior, causes bugs in many cases)"), HIVE_MSCK_REPAIR_BATCH_SIZE( "hive.msck.repair.batch.size", 0, -"Batch size for the msck repair command. If the value is greater than zero, " -+ "it will execute batch wise with the configured batch size. " -+ "The default value is zero. Zero means it will execute directly (Not batch wise)"), +"Batch size for the msck repair command. If the value is greater than zero,\n " ++ "it will execute batch wise with the configured batch size. In case of errors while\n" ++ "adding unknown partitions the batch size is automatically reduced by half in the subsequent\n" ++ "retry attempt. The default value is zero which means it will execute directly (not batch wise)"), +HIVE_MSCK_REPAIR_BATCH_MAX_RETRIES("hive.msck.repair.batch.max.retries", 0, +"Maximum number of retries for the msck repair command when adding unknown partitions.\n " ++ "If the value is greater than zero it will retry adding unknown partitions until the maximum\n" ++ "number of attempts is reached or batch size is reduced to 0, whichever is earlier.\n" ++ "In each retry attempt it will reduce the batch size by a factor of 2 until it reaches zero.\n" ++ "If the value is set to zero it will retry until the batch size becomes zero as described above."), HIVE_SERVER2_LLAP_CONCURRENT_QUERIES("hive.server2.llap.concurrent.queries", -1, "The number of queries allowed in parallel via llap. Negative number implies 'infinite'."), HIVE_TEZ_ENABLE_MEMORY_MANAGER("hive.tez.enable.memory.manager", true, http://git-wip-us.apache.org/repos/asf/hive/blob/0efb9368/common/src/java/org/apache/hive/common/util/RetryUtilities.java -- diff --git a/common/src/java/org/apache/hive/common/util/RetryUtilities.java b/common/src/java/org/apache/hive/common/util/RetryUtilities.java new file mode 100644 index 000..3a20f2c --- /dev/null +++ b/common/src/java/org/apache/hive/common/util/RetryUtilities.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file
[1/2] hive git commit: HIVE-16645: Commands.java has missed the catch statement and has some code format errors (Saijin Huang, reviewed by Aihua Xu & Peter Vary)
Repository: hive Updated Branches: refs/heads/master 77f44b66d -> 86f74fdd2 HIVE-16645: Commands.java has missed the catch statement and has some code format errors (Saijin Huang, reviewed by Aihua Xu & Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/455ffdd9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/455ffdd9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/455ffdd9 Branch: refs/heads/master Commit: 455ffdd9125bdfe73b2c7f7ddebaeff138b77f53 Parents: 77f44b6 Author: Aihua Xu Authored: Mon May 15 11:09:08 2017 -0400 Committer: Aihua Xu Committed: Mon May 15 11:36:07 2017 -0400 -- .../src/java/org/apache/hive/beeline/Commands.java | 16 1 file changed, 12 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/455ffdd9/beeline/src/java/org/apache/hive/beeline/Commands.java -- diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index da896a7..bcbc574 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -304,6 +304,7 @@ public class Commands { try { rs.close(); } catch (Exception e) { + beeLine.error(e); } } // run as a batch @@ -787,6 +788,7 @@ public class Commands { private BufferedRows getConfInternal(boolean call) { Statement stmnt = null; BufferedRows rows = null; +ResultSet rs = null; try { boolean hasResults = false; DatabaseConnection dbconn = beeLine.getDatabaseConnection(); @@ -803,19 +805,26 @@ public class Commands { } } if (hasResults) { -ResultSet rs = stmnt.getResultSet(); +rs = stmnt.getResultSet(); rows = new BufferedRows(beeLine, rs); } } catch (SQLException e) { beeLine.error(e); } finally { - if (stmnt != null) { + if (rs != null) { try { - stmnt.close(); + rs.close(); } catch (SQLException e1) { beeLine.error(e1); } } + if (stmnt != null) { +try { + stmnt.close(); +} catch (SQLException e2) { + beeLine.error(e2); +} + } } return rows; } @@ -1428,7 +1437,6 @@ public class Commands { public boolean closeall(String line) { if (close(null)) { while (close(null)) { -; } return true; }
[2/2] hive git commit: HIVE-16607: ColumnStatsAutoGatherContext regenerates HiveConf.HIVEQUERYID (Peter Vary, reviewed by Aihua Xu)
HIVE-16607: ColumnStatsAutoGatherContext regenerates HiveConf.HIVEQUERYID (Peter Vary, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/86f74fdd Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/86f74fdd Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/86f74fdd Branch: refs/heads/master Commit: 86f74fdd26b83cee128d24055ee369e9b7f36201 Parents: 455ffdd Author: Aihua Xu Authored: Mon May 15 11:12:04 2017 -0400 Committer: Aihua Xu Committed: Mon May 15 11:36:22 2017 -0400 -- .../mapreduce/TestHCatMultiOutputFormat.java| 2 +- .../test/resources/testconfiguration.properties | 1 + .../org/apache/hadoop/hive/ql/QTestUtil.java| 4 +- .../java/org/apache/hive/beeline/QFile.java | 20 +- .../apache/hive/beeline/QFileBeeLineClient.java | 8 +- .../java/org/apache/hadoop/hive/ql/Driver.java | 28 +- .../org/apache/hadoop/hive/ql/QueryState.java | 146 ++--- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 2 +- .../ql/io/rcfile/stats/PartialScanTask.java | 3 +- .../metadata/HiveMaterializedViewsRegistry.java | 5 +- .../ql/parse/ColumnStatsAutoGatherContext.java | 9 +- .../hadoop/hive/ql/exec/TestExecDriver.java | 3 +- .../ql/parse/TestMacroSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/TestQBCompact.java | 2 +- .../ql/parse/TestQBJoinTreeApplyPredicate.java | 3 +- .../hadoop/hive/ql/parse/TestQBSubQuery.java| 3 +- .../parse/TestReplicationSemanticAnalyzer.java | 3 +- .../ql/parse/TestSemanticAnalyzerFactory.java | 2 +- .../parse/TestUpdateDeleteSemanticAnalyzer.java | 2 +- .../TestHiveAuthorizationTaskFactory.java | 2 +- .../parse/authorization/TestPrivilegesV1.java | 4 +- .../parse/authorization/TestPrivilegesV2.java | 2 +- .../materialized_view_create_rewrite.q.out | 322 +++ .../hive/service/cli/operation/Operation.java | 12 +- .../service/cli/operation/SQLOperation.java | 6 +- 25 files changed, 498 insertions(+), 98 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/86f74fdd/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java -- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index 6ff48ee..180e802 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -356,7 +356,7 @@ public class TestHCatMultiOutputFormat { * @throws Exception if any error occurs */ private List getTableData(String table, String database) throws Exception { -QueryState queryState = new QueryState(null); +QueryState queryState = new QueryState.Builder().build(); HiveConf conf = queryState.getConf(); conf.addResource("hive-site.xml"); ArrayList results = new ArrayList(); http://git-wip-us.apache.org/repos/asf/hive/blob/86f74fdd/itests/src/test/resources/testconfiguration.properties -- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index a378a5d..51385cf 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -756,6 +756,7 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\ beeline.positive.include=drop_with_concurrency.q,\ escape_comments.q,\ mapjoin2.q,\ + materialized_view_create_rewrite.q,\ smb_mapjoin_1.q,\ smb_mapjoin_10.q,\ smb_mapjoin_11.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/86f74fdd/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java -- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index b897ffa..d408321 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -566,7 +566,7 @@ public class QTestUtil { System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation()); } -queryState = new QueryState(new HiveConf(Driver.class)); +queryState = new QueryState.Builder().withHiveConf(new HiveConf(Driver.class)).build(); if (useHBaseMetastore) { startMiniHBaseCluster(); } else { @@ -1896,7 +1896,7 @@ publi
hive git commit: HIVE-16577: Syntax error in the metastore init scripts for mssql (Vihang Karajgaonkar, reviewed by Aihua Xu & Thejas M Nair)
Repository: hive Updated Branches: refs/heads/branch-2.3 3fbdca546 -> a4214f1a3 HIVE-16577: Syntax error in the metastore init scripts for mssql (Vihang Karajgaonkar, reviewed by Aihua Xu & Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a4214f1a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a4214f1a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a4214f1a Branch: refs/heads/branch-2.3 Commit: a4214f1a34b0a51110ab7e734db26fff67f2a1c3 Parents: 3fbdca5 Author: Aihua Xu Authored: Fri May 5 14:00:51 2017 -0400 Committer: Aihua Xu Committed: Mon May 8 13:51:55 2017 -0400 -- metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql | 2 +- metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/a4214f1a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql index 57dd30f..33730de 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql @@ -579,7 +579,7 @@ CREATE TABLE NOTIFICATION_LOG EVENT_TYPE nvarchar(32) NOT NULL, DB_NAME nvarchar(128) NULL, TBL_NAME nvarchar(256) NULL, -MESSAGE_FORMAT nvarchar(16) +MESSAGE_FORMAT nvarchar(16), MESSAGE text NULL ); http://git-wip-us.apache.org/repos/asf/hive/blob/a4214f1a/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql index 8a80a50..c117a32 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql @@ -579,7 +579,7 @@ CREATE TABLE NOTIFICATION_LOG EVENT_TYPE nvarchar(32) NOT NULL, DB_NAME nvarchar(128) NULL, TBL_NAME nvarchar(256) NULL, -MESSAGE_FORMAT nvarchar(16) +MESSAGE_FORMAT nvarchar(16), MESSAGE text NULL );
hive git commit: HIVE-16450: Some metastore operations are not retried even with desired underlining exceptions (Aihua Xu, reviewed by Naveen Gangam & Peter Vary)
Repository: hive Updated Branches: refs/heads/master 54dbca69c -> 301e7c5ea HIVE-16450: Some metastore operations are not retried even with desired underlining exceptions (Aihua Xu, reviewed by Naveen Gangam & Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/301e7c5e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/301e7c5e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/301e7c5e Branch: refs/heads/master Commit: 301e7c5eaba790687818a57d92b046f746bb3d76 Parents: 54dbca6 Author: Aihua Xu Authored: Fri Apr 14 10:53:58 2017 -0400 Committer: Aihua Xu Committed: Mon May 8 10:20:47 2017 -0400 -- .../apache/hadoop/hive/metastore/Deadline.java | 29 ++--- .../hadoop/hive/metastore/MetaStoreUtils.java | 24 ++ .../hadoop/hive/metastore/ObjectStore.java | 34 .../hive/metastore/RetryingHMSHandler.java | 2 -- 4 files changed, 47 insertions(+), 42 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/301e7c5e/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java -- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java index 71d336a..6149224 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Deadline.java @@ -86,15 +86,15 @@ public class Deadline { */ public static void resetTimeout(long timeoutMs) throws MetaException { if (timeoutMs <= 0) { - throw newMetaException(new DeadlineException("The reset timeout value should be " + + throw MetaStoreUtils.newMetaException(new DeadlineException("The reset timeout value should be " + "larger than 0: " + timeoutMs)); } Deadline deadline = getCurrentDeadline(); if (deadline != null) { deadline.timeoutNanos = timeoutMs * 100L; } else { - throw newMetaException(new DeadlineException("The threadlocal Deadline is null," + - " please register it firstly.")); + throw MetaStoreUtils.newMetaException(new DeadlineException("The threadlocal Deadline is null," + + " please register it first.")); } } @@ -105,8 +105,8 @@ public class Deadline { public static boolean startTimer(String method) throws MetaException { Deadline deadline = getCurrentDeadline(); if (deadline == null) { - throw newMetaException(new DeadlineException("The threadlocal Deadline is null," + - " please register it firstly.")); + throw MetaStoreUtils.newMetaException(new DeadlineException("The threadlocal Deadline is null," + + " please register it first.")); } if (deadline.startTime != NO_DEADLINE) return false; deadline.method = method; @@ -125,8 +125,8 @@ public class Deadline { deadline.startTime = NO_DEADLINE; deadline.method = null; } else { - throw newMetaException(new DeadlineException("The threadlocal Deadline is null," + - " please register it firstly.")); + throw MetaStoreUtils.newMetaException(new DeadlineException("The threadlocal Deadline is null," + + " please register it first.")); } } @@ -146,7 +146,7 @@ public class Deadline { if (deadline != null) { deadline.check(); } else { - throw newMetaException(new DeadlineException("The threadlocal Deadline is null," + + throw MetaStoreUtils.newMetaException(new DeadlineException("The threadlocal Deadline is null," + " please register it first.")); } } @@ -165,18 +165,7 @@ public class Deadline { + (elapsedTime / 100L) + "ms exceeds " + (timeoutNanos / 100L) + "ms"); } } catch (DeadlineException e) { - throw newMetaException(e); + throw MetaStoreUtils.newMetaException(e); } } - - /** - * convert DeadlineException to MetaException - * @param e - * @return - */ - private static MetaException newMetaException(DeadlineException e) { -MetaException metaException = new MetaException(e.getMessage()); -metaException.initCause(e); -return metaException; - } } http://git-wip-us.apache.org/repos/asf/hive/blob/301e7c5e/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java -- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index d67e03f..870896c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive
hive git commit: HIVE-16577: Syntax error in the metastore init scripts for mssql (Vihang Karajgaonkar, reviewed by Aihua Xu & Thejas M Nair)
Repository: hive Updated Branches: refs/heads/branch-2 08d559b15 -> afc3dabf5 HIVE-16577: Syntax error in the metastore init scripts for mssql (Vihang Karajgaonkar, reviewed by Aihua Xu & Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/afc3dabf Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/afc3dabf Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/afc3dabf Branch: refs/heads/branch-2 Commit: afc3dabf5eb3568d7372bb621c2d00adeac52a2d Parents: 08d559b Author: Aihua Xu Authored: Fri May 5 14:00:51 2017 -0400 Committer: Aihua Xu Committed: Fri May 5 14:03:00 2017 -0400 -- metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql | 2 +- metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/afc3dabf/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql index 57dd30f..33730de 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql @@ -579,7 +579,7 @@ CREATE TABLE NOTIFICATION_LOG EVENT_TYPE nvarchar(32) NOT NULL, DB_NAME nvarchar(128) NULL, TBL_NAME nvarchar(256) NULL, -MESSAGE_FORMAT nvarchar(16) +MESSAGE_FORMAT nvarchar(16), MESSAGE text NULL ); http://git-wip-us.apache.org/repos/asf/hive/blob/afc3dabf/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql index 8a80a50..c117a32 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql @@ -579,7 +579,7 @@ CREATE TABLE NOTIFICATION_LOG EVENT_TYPE nvarchar(32) NOT NULL, DB_NAME nvarchar(128) NULL, TBL_NAME nvarchar(256) NULL, -MESSAGE_FORMAT nvarchar(16) +MESSAGE_FORMAT nvarchar(16), MESSAGE text NULL );
hive git commit: HIVE-16577: Syntax error in the metastore init scripts for mssql (Vihang Karajgaonkar, reviewed by Aihua Xu & Thejas M Nair)
Repository: hive Updated Branches: refs/heads/master 1fecb81f9 -> d09f3f81d HIVE-16577: Syntax error in the metastore init scripts for mssql (Vihang Karajgaonkar, reviewed by Aihua Xu & Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d09f3f81 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d09f3f81 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d09f3f81 Branch: refs/heads/master Commit: d09f3f81d231e68727bbb39681a686c5f525114a Parents: 1fecb81 Author: Aihua Xu Authored: Fri May 5 14:00:51 2017 -0400 Committer: Aihua Xu Committed: Fri May 5 14:00:51 2017 -0400 -- metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql | 2 +- metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql | 2 +- metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/d09f3f81/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql index 57dd30f..33730de 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql @@ -579,7 +579,7 @@ CREATE TABLE NOTIFICATION_LOG EVENT_TYPE nvarchar(32) NOT NULL, DB_NAME nvarchar(128) NULL, TBL_NAME nvarchar(256) NULL, -MESSAGE_FORMAT nvarchar(16) +MESSAGE_FORMAT nvarchar(16), MESSAGE text NULL ); http://git-wip-us.apache.org/repos/asf/hive/blob/d09f3f81/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql index 8a80a50..c117a32 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-2.3.0.mssql.sql @@ -579,7 +579,7 @@ CREATE TABLE NOTIFICATION_LOG EVENT_TYPE nvarchar(32) NOT NULL, DB_NAME nvarchar(128) NULL, TBL_NAME nvarchar(256) NULL, -MESSAGE_FORMAT nvarchar(16) +MESSAGE_FORMAT nvarchar(16), MESSAGE text NULL ); http://git-wip-us.apache.org/repos/asf/hive/blob/d09f3f81/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql -- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql index 98682a8..54d593c 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql @@ -579,7 +579,7 @@ CREATE TABLE NOTIFICATION_LOG EVENT_TYPE nvarchar(32) NOT NULL, DB_NAME nvarchar(128) NULL, TBL_NAME nvarchar(256) NULL, -MESSAGE_FORMAT nvarchar(16) +MESSAGE_FORMAT nvarchar(16), MESSAGE text NULL );
hive git commit: HIVE-16335: Beeline user HS2 connection file should use /etc/hive/conf instead of /etc/conf/hive (Vihang Karajgaonkar, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/branch-2 242180be3 -> 08d559b15 HIVE-16335: Beeline user HS2 connection file should use /etc/hive/conf instead of /etc/conf/hive (Vihang Karajgaonkar, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/08d559b1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/08d559b1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/08d559b1 Branch: refs/heads/branch-2 Commit: 08d559b1536064ef0e666e5baab78bb6b1b28159 Parents: 242180b Author: Aihua Xu Authored: Tue Apr 4 19:12:50 2017 -0400 Committer: Aihua Xu Committed: Thu May 4 15:22:46 2017 -0400 -- .../hive/beeline/hs2connection/UserHS2ConnectionFileParser.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/08d559b1/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java -- diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java index 93a6231..7d7d9ae 100644 --- a/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java @@ -44,7 +44,7 @@ public class UserHS2ConnectionFileParser implements HS2ConnectionFileParser { + (System.getProperty("os.name").toLowerCase().indexOf("windows") != -1 ? "" : ".") + "beeline" + File.separator; public static final String ETC_HIVE_CONF_LOCATION = - File.separator + "etc" + File.separator + "conf" + File.separator + "hive"; + File.separator + "etc" + File.separator + "hive" + File.separator + "conf"; private final List locations = new ArrayList<>(); private static final Logger log = LoggerFactory.getLogger(UserHS2ConnectionFileParser.class);
hive git commit: HIVE-16346: inheritPerms should be conditional based on the target filesystem (Sahil Takiar, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/branch-2 ec5b137ab -> 7b1f6efa5 HIVE-16346: inheritPerms should be conditional based on the target filesystem (Sahil Takiar, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7b1f6efa Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7b1f6efa Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7b1f6efa Branch: refs/heads/branch-2 Commit: 7b1f6efa52a62ea92e8e33c1db9050f38fdeec9c Parents: ec5b137 Author: Aihua Xu Authored: Fri Apr 28 13:26:55 2017 -0400 Committer: Aihua Xu Committed: Fri Apr 28 13:39:22 2017 -0400 -- common/pom.xml | 8 +- .../hadoop/hive/common/BlobStorageUtils.java| 10 +- .../apache/hadoop/hive/common/FileUtils.java| 39 ++- .../apache/hadoop/hive/common/StorageUtils.java | 40 +++ .../org/apache/hadoop/hive/io/HdfsUtils.java| 248 +++ .../hive/common/TestBlobStorageUtils.java | 8 +- .../hadoop/hive/common/TestStorageUtils.java| 57 + .../apache/hadoop/hive/io/TestHdfsUtils.java| 203 +++ .../apache/hadoop/hive/metastore/Warehouse.java | 5 +- .../java/org/apache/hadoop/hive/ql/Context.java | 4 +- .../apache/hadoop/hive/ql/exec/CopyTask.java| 3 +- .../apache/hadoop/hive/ql/exec/MoveTask.java| 4 +- .../hadoop/hive/ql/exec/ReplCopyTask.java | 3 +- .../apache/hadoop/hive/ql/metadata/Hive.java| 29 +-- .../org/apache/hadoop/hive/io/HdfsUtils.java| 239 -- .../apache/hadoop/hive/io/TestHdfsUtils.java| 184 -- 16 files changed, 621 insertions(+), 463 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/7b1f6efa/common/pom.xml -- diff --git a/common/pom.xml b/common/pom.xml index e1c15ee..84bb1e5 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -157,7 +157,13 @@ commons-logging - + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + true + com.google.code.tempus-fugit http://git-wip-us.apache.org/repos/asf/hive/blob/7b1f6efa/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java -- diff --git a/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java b/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java index e6a17cb..b7f1359 100644 --- a/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.hadoop.hive.common; import org.apache.hadoop.conf.Configuration; @@ -24,21 +25,23 @@ import org.apache.hadoop.hive.conf.HiveConf; import java.util.Collection; + /** * Utilities for different blob (object) storage systems */ public class BlobStorageUtils { + private static final boolean DISABLE_BLOBSTORAGE_AS_SCRATCHDIR = false; public static boolean isBlobStoragePath(final Configuration conf, final Path path) { return path != null && isBlobStorageScheme(conf, path.toUri().getScheme()); } -public static boolean isBlobStorageFileSystem(final Configuration conf, final FileSystem fs) { -return fs != null && isBlobStorageScheme(conf, fs.getScheme()); +static boolean isBlobStorageFileSystem(final Configuration conf, final FileSystem fs) { +return fs != null && fs.getUri() != null && isBlobStorageScheme(conf, fs.getUri().getScheme()); } -public static boolean isBlobStorageScheme(final Configuration conf, final String scheme) { +static boolean isBlobStorageScheme(final Configuration conf, final String scheme) { Collection supportedBlobStoreSchemes = conf.getStringCollection(HiveConf.ConfVars.HIVE_BLOBSTORE_SUPPORTED_SCHEMES.varname); @@ -61,4 +64,5 @@ public class BlobStorageUtils { HiveConf.ConfVars.HIVE_BLOBSTORE_OPTIMIZATIONS_ENABLED.defaultBoolVal ); } + } http://git-wip-us.apache.org/repos/asf/hive/blob/7b1f6efa/common/src/java/org/apache/hadoop/hive/common/FileUtils.java -- diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java index e586015..8ed8cc4 100644 --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -46,13 +46,11 @@ import org.apache.hadoop.fs.PathFilter;
hive git commit: Revert "HIVE-16346: inheritPerms should be conditional based on the target filesystem (Sahil Takiar, reviewed by Aihua Xu)"
Repository: hive Updated Branches: refs/heads/branch-2 ab3a24bf6 -> 2ba121757 Revert "HIVE-16346: inheritPerms should be conditional based on the target filesystem (Sahil Takiar, reviewed by Aihua Xu)" This reverts commit cce4d5e78582c8744972d265147d39a345e082db. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2ba12175 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2ba12175 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2ba12175 Branch: refs/heads/branch-2 Commit: 2ba121757699c551a7688f1e42c0d4ecedc826f8 Parents: ab3a24b Author: Aihua Xu Authored: Fri Apr 28 11:22:22 2017 -0400 Committer: Aihua Xu Committed: Fri Apr 28 11:22:22 2017 -0400 -- common/pom.xml | 8 +-- .../hadoop/hive/common/BlobStorageUtils.java| 10 +-- .../apache/hadoop/hive/common/FileUtils.java| 39 ++- .../apache/hadoop/hive/common/StorageUtils.java | 40 --- .../hive/common/TestBlobStorageUtils.java | 8 +-- .../hadoop/hive/common/TestStorageUtils.java| 57 --- .../apache/hadoop/hive/metastore/Warehouse.java | 5 +- .../java/org/apache/hadoop/hive/ql/Context.java | 4 +- .../apache/hadoop/hive/ql/exec/CopyTask.java| 3 +- .../apache/hadoop/hive/ql/exec/MoveTask.java| 4 +- .../hadoop/hive/ql/exec/ReplCopyTask.java | 3 +- .../apache/hadoop/hive/ql/metadata/Hive.java| 29 .../org/apache/hadoop/hive/io/HdfsUtils.java| 73 +--- .../apache/hadoop/hive/io/TestHdfsUtils.java| 19 - 14 files changed, 72 insertions(+), 230 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/2ba12175/common/pom.xml -- diff --git a/common/pom.xml b/common/pom.xml index 84bb1e5..e1c15ee 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -157,13 +157,7 @@ commons-logging - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - true - + com.google.code.tempus-fugit http://git-wip-us.apache.org/repos/asf/hive/blob/2ba12175/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java -- diff --git a/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java b/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java index b7f1359..e6a17cb 100644 --- a/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hive.common; import org.apache.hadoop.conf.Configuration; @@ -25,23 +24,21 @@ import org.apache.hadoop.hive.conf.HiveConf; import java.util.Collection; - /** * Utilities for different blob (object) storage systems */ public class BlobStorageUtils { - private static final boolean DISABLE_BLOBSTORAGE_AS_SCRATCHDIR = false; public static boolean isBlobStoragePath(final Configuration conf, final Path path) { return path != null && isBlobStorageScheme(conf, path.toUri().getScheme()); } -static boolean isBlobStorageFileSystem(final Configuration conf, final FileSystem fs) { -return fs != null && fs.getUri() != null && isBlobStorageScheme(conf, fs.getUri().getScheme()); +public static boolean isBlobStorageFileSystem(final Configuration conf, final FileSystem fs) { +return fs != null && isBlobStorageScheme(conf, fs.getScheme()); } -static boolean isBlobStorageScheme(final Configuration conf, final String scheme) { +public static boolean isBlobStorageScheme(final Configuration conf, final String scheme) { Collection supportedBlobStoreSchemes = conf.getStringCollection(HiveConf.ConfVars.HIVE_BLOBSTORE_SUPPORTED_SCHEMES.varname); @@ -64,5 +61,4 @@ public class BlobStorageUtils { HiveConf.ConfVars.HIVE_BLOBSTORE_OPTIMIZATIONS_ENABLED.defaultBoolVal ); } - } http://git-wip-us.apache.org/repos/asf/hive/blob/2ba12175/common/src/java/org/apache/hadoop/hive/common/FileUtils.java -- diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java index 8ed8cc4..e586015 100644 --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -46,11 +46,13 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.f
hive git commit: HIVE-16346: inheritPerms should be conditional based on the target filesystem (Sahil Takiar, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/branch-2 59faf3695 -> cce4d5e78 HIVE-16346: inheritPerms should be conditional based on the target filesystem (Sahil Takiar, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cce4d5e7 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cce4d5e7 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cce4d5e7 Branch: refs/heads/branch-2 Commit: cce4d5e78582c8744972d265147d39a345e082db Parents: 59faf36 Author: Aihua Xu Authored: Mon Apr 24 16:40:57 2017 -0400 Committer: Aihua Xu Committed: Mon Apr 24 16:40:57 2017 -0400 -- common/pom.xml | 8 ++- .../hadoop/hive/common/BlobStorageUtils.java| 10 ++- .../apache/hadoop/hive/common/FileUtils.java| 39 +-- .../apache/hadoop/hive/common/StorageUtils.java | 40 +++ .../hive/common/TestBlobStorageUtils.java | 8 +-- .../hadoop/hive/common/TestStorageUtils.java| 57 +++ .../apache/hadoop/hive/metastore/Warehouse.java | 5 +- .../java/org/apache/hadoop/hive/ql/Context.java | 4 +- .../apache/hadoop/hive/ql/exec/CopyTask.java| 3 +- .../apache/hadoop/hive/ql/exec/MoveTask.java| 4 +- .../hadoop/hive/ql/exec/ReplCopyTask.java | 3 +- .../apache/hadoop/hive/ql/metadata/Hive.java| 29 .../org/apache/hadoop/hive/io/HdfsUtils.java| 73 +++- .../apache/hadoop/hive/io/TestHdfsUtils.java| 19 + 14 files changed, 230 insertions(+), 72 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/cce4d5e7/common/pom.xml -- diff --git a/common/pom.xml b/common/pom.xml index e1c15ee..84bb1e5 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -157,7 +157,13 @@ commons-logging - + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + true + com.google.code.tempus-fugit http://git-wip-us.apache.org/repos/asf/hive/blob/cce4d5e7/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java -- diff --git a/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java b/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java index e6a17cb..b7f1359 100644 --- a/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/BlobStorageUtils.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.hadoop.hive.common; import org.apache.hadoop.conf.Configuration; @@ -24,21 +25,23 @@ import org.apache.hadoop.hive.conf.HiveConf; import java.util.Collection; + /** * Utilities for different blob (object) storage systems */ public class BlobStorageUtils { + private static final boolean DISABLE_BLOBSTORAGE_AS_SCRATCHDIR = false; public static boolean isBlobStoragePath(final Configuration conf, final Path path) { return path != null && isBlobStorageScheme(conf, path.toUri().getScheme()); } -public static boolean isBlobStorageFileSystem(final Configuration conf, final FileSystem fs) { -return fs != null && isBlobStorageScheme(conf, fs.getScheme()); +static boolean isBlobStorageFileSystem(final Configuration conf, final FileSystem fs) { +return fs != null && fs.getUri() != null && isBlobStorageScheme(conf, fs.getUri().getScheme()); } -public static boolean isBlobStorageScheme(final Configuration conf, final String scheme) { +static boolean isBlobStorageScheme(final Configuration conf, final String scheme) { Collection supportedBlobStoreSchemes = conf.getStringCollection(HiveConf.ConfVars.HIVE_BLOBSTORE_SUPPORTED_SCHEMES.varname); @@ -61,4 +64,5 @@ public class BlobStorageUtils { HiveConf.ConfVars.HIVE_BLOBSTORE_OPTIMIZATIONS_ENABLED.defaultBoolVal ); } + } http://git-wip-us.apache.org/repos/asf/hive/blob/cce4d5e7/common/src/java/org/apache/hadoop/hive/common/FileUtils.java -- diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java index e586015..8ed8cc4 100644 --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -46,13 +46,11 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hive.conf.HiveConf; -im
[1/2] hive git commit: HIVE-15442: Driver.java has a redundancy code (Saijin Huang, reviewed by Aihua Xu)
Repository: hive Updated Branches: refs/heads/master 98250bbe6 -> 99f142c9a HIVE-15442: Driver.java has a redundancy code (Saijin Huang, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/579f5d7e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/579f5d7e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/579f5d7e Branch: refs/heads/master Commit: 579f5d7e6ef87f108e5c482f30f72e612ef571ef Parents: 98250bb Author: Aihua Xu Authored: Mon Apr 17 09:25:39 2017 -0400 Committer: Aihua Xu Committed: Mon Apr 17 09:25:39 2017 -0400 -- ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 6 ++ 1 file changed, 2 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/579f5d7e/ql/src/java/org/apache/hadoop/hive/ql/Driver.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index a2ce71d..03ea997 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -561,10 +561,8 @@ public class Driver implements CommandProcessor { if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) { String explainOutput = getExplainOutput(sem, plan, tree); if (explainOutput != null) { - if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) { -LOG.info("EXPLAIN output for queryid " + queryId + " : " - + explainOutput); - } + LOG.info("EXPLAIN output for queryid " + queryId + " : " ++ explainOutput); if (conf.isWebUiQueryInfoCacheEnabled()) { queryDisplay.setExplainPlan(explainOutput); }
[2/2] hive git commit: HIVE-16409: TestEventHandlerFactory has lacked the ASF header (Saijin Huang, reviewed by Aihua Xu)
HIVE-16409: TestEventHandlerFactory has lacked the ASF header (Saijin Huang, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/99f142c9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/99f142c9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/99f142c9 Branch: refs/heads/master Commit: 99f142c9a010e74843497d813652c3ea44ad3085 Parents: 579f5d7 Author: Aihua Xu Authored: Mon Apr 17 09:26:39 2017 -0400 Committer: Aihua Xu Committed: Mon Apr 17 09:26:39 2017 -0400 -- .../repl/events/TestEventHandlerFactory.java | 18 ++ 1 file changed, 18 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/99f142c9/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java -- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java index 0526700..d44cb79 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.ql.parse.repl.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent;
[3/4] hive git commit: HIVE-16400: Fix the MDC reference to use slf4j rather than log4j (Aihua Xu, reviewed by Yongzhi Chen & Naveen Gangam)
HIVE-16400: Fix the MDC reference to use slf4j rather than log4j (Aihua Xu, reviewed by Yongzhi Chen & Naveen Gangam) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f8404d84 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f8404d84 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f8404d84 Branch: refs/heads/master Commit: f8404d846a6f1c48507e16d0d33d75d726d169f3 Parents: c926f81 Author: Aihua Xu Authored: Thu Apr 6 12:49:25 2017 -0400 Committer: Aihua Xu Committed: Fri Apr 7 09:55:41 2017 -0400 -- common/src/java/org/apache/hadoop/hive/common/LogUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/f8404d84/common/src/java/org/apache/hadoop/hive/common/LogUtils.java -- diff --git a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java index 1d6b55e..83f3af7 100644 --- a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java @@ -24,13 +24,13 @@ import java.net.URL; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.log4j.MDC; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.core.impl.Log4jContextFactory; import org.apache.logging.log4j.spi.DefaultThreadContextMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import com.google.common.annotations.VisibleForTesting;
[2/4] hive git commit: HIVE-16340: Allow Kerberos + SSL connections to HMS (Sahil Takiar, reviewed by Aihua Xu)
HIVE-16340: Allow Kerberos + SSL connections to HMS (Sahil Takiar, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c926f81e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c926f81e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c926f81e Branch: refs/heads/master Commit: c926f81e04a7bc49487ea4ef0e9d3a273d60f8aa Parents: b466953 Author: Aihua Xu Authored: Fri Apr 7 09:48:05 2017 -0400 Committer: Aihua Xu Committed: Fri Apr 7 09:48:05 2017 -0400 -- .../apache/hive/minikdc/TestSSLWithMiniKdc.java | 101 + .../java/org/hadoop/hive/jdbc/SSLTestUtils.java | 103 + .../test/java/org/apache/hive/jdbc/TestSSL.java | 150 ++- .../hadoop/hive/metastore/HiveMetaStore.java| 32 ++-- .../hive/metastore/HiveMetaStoreClient.java | 51 --- .../hadoop/hive/metastore/MetaStoreUtils.java | 15 +- 6 files changed, 307 insertions(+), 145 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/c926f81e/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestSSLWithMiniKdc.java -- diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestSSLWithMiniKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestSSLWithMiniKdc.java new file mode 100644 index 000..3153b9f --- /dev/null +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestSSLWithMiniKdc.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.minikdc; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.hadoop.hive.jdbc.SSLTestUtils; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + + +public class TestSSLWithMiniKdc { + + private static MiniHS2 miniHS2 = null; + private static MiniHiveKdc miniHiveKdc = null; + + @BeforeClass + public static void beforeTest() throws Exception { +Class.forName(MiniHS2.getJdbcDriverName()); + +HiveConf hiveConf = new HiveConf(); + +SSLTestUtils.setMetastoreSslConf(hiveConf); +hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + +miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf); + +setHMSSaslConf(miniHiveKdc, hiveConf); + +miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf); + +Map confOverlay = new HashMap<>(); +SSLTestUtils.setHttpConfOverlay(confOverlay); +SSLTestUtils.setSslConfOverlay(confOverlay); + +miniHS2.start(confOverlay); + } + + @AfterClass + public static void afterTest() throws Exception { +miniHS2.stop(); + } + + @Test + public void testConnection() throws Exception { +String tableName = "testTable"; +Path dataFilePath = new Path(SSLTestUtils.getDataFileDir(), "kv1.txt"); +Connection hs2Conn = getConnection(MiniHiveKdc.HIVE_TEST_USER_1); + +Statement stmt = hs2Conn.createStatement(); + +SSLTestUtils.setupTestTableWithData(tableName, dataFilePath, hs2Conn); + +stmt.execute("select * from " + tableName); +stmt.execute("drop table " + tableName); +stmt.close(); + } + + private Connection getConnection(String userName) throws Exception { +miniHiveKdc.loginUser(userName); +return DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS), +System.getProperty("user.name"), "bar"); + } + + private static void setHMSSaslConf(MiniHiveKdc miniHiveKdc, HiveConf conf) { + String hivePrincipal = + miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL); +String hiveKeytab = miniHiveKdc.getKeyTabFile( + miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_SERV
[4/4] hive git commit: HIVE-16291: Hive fails when unions a parquet table with itself (Yibing Shi, reviewed by Ashutosh Chauhan & Aihua Xu)
HIVE-16291: Hive fails when unions a parquet table with itself (Yibing Shi, reviewed by Ashutosh Chauhan & Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a5f94c07 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a5f94c07 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a5f94c07 Branch: refs/heads/master Commit: a5f94c0710ac6ad04305e2cc320bb9eede301815 Parents: f8404d8 Author: Aihua Xu Authored: Fri Apr 7 09:57:25 2017 -0400 Committer: Aihua Xu Committed: Fri Apr 7 09:57:25 2017 -0400 -- .../hadoop/hive/serde2/ColumnProjectionUtils.java | 6 ++ .../hadoop/hive/serde2/TestColumnProjectionUtils.java | 14 ++ 2 files changed, 16 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/a5f94c07/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java -- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java index 1354680..9844166 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; +import org.apache.hive.common.util.HiveStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,10 +116,7 @@ public final class ColumnProjectionUtils { public static void appendReadColumns(Configuration conf, List ids) { String id = toReadColumnIDString(ids); String old = conf.get(READ_COLUMN_IDS_CONF_STR, null); -String newConfStr = id; -if (old != null && !old.isEmpty()) { - newConfStr = newConfStr + StringUtils.COMMA_STR + old; -} +String newConfStr = HiveStringUtils.joinIgnoringEmpty(new String[] {id, old}, StringUtils.COMMA); setReadColumnIDConf(conf, newConfStr); // Set READ_ALL_COLUMNS to false conf.setBoolean(READ_ALL_COLUMNS, false); http://git-wip-us.apache.org/repos/asf/hive/blob/a5f94c07/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java -- diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java b/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java index 2b81b54..fbd9b07 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java @@ -85,6 +85,20 @@ public class TestColumnProjectionUtils { } @Test + public void testMultipleIdsWithEmpty() { +List ids1 = Arrays.asList(1, 2); +List ids2 = new ArrayList(); +List ids3 = Arrays.asList(2, 3); + +ColumnProjectionUtils.appendReadColumns(conf, ids1); +ColumnProjectionUtils.appendReadColumns(conf, ids2); +ColumnProjectionUtils.appendReadColumns(conf, ids3); + +List actual = ColumnProjectionUtils.getReadColumnIDs(conf); +assertEquals(Arrays.asList(2, 3, 1), actual); + } + + @Test public void testDeprecatedMethods() { List columnIds = new ArrayList(); List actual;