[hive] branch master updated: HIVE-23961:Enable external table replication by default (Aasha Medhi, reviewed by Pravin Kumar Sinha)
This is an automated email from the ASF dual-hosted git repository.
anishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5bc50f6 HIVE-23961:Enable external table replication by default
(Aasha Medhi, reviewed by Pravin Kumar Sinha)
5bc50f6 is described below
commit 5bc50f6afda439002f63c77fa7e013f64279e4b0
Author: Anishek Agarwal
AuthorDate: Wed Aug 5 09:14:56 2020 +0530
HIVE-23961:Enable external table replication by default (Aasha Medhi,
reviewed by Pravin Kumar Sinha)
---
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 6 +++---
.../hive/metastore/TestMetaStoreMultipleEncryptionZones.java | 8
.../org/apache/hadoop/hive/metastore/TestReplChangeManager.java | 8
.../java/org/apache/hadoop/hive/ql/parse/TestExportImport.java| 1 +
.../hive/ql/parse/TestReplicationScenariosAcrossInstances.java| 4 ++--
ql/src/test/queries/clientpositive/exim_10_external_managed.q | 1 +
.../java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java | 2 +-
7 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index bba2656..9b637fb 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -511,8 +511,8 @@ public class HiveConf extends Configuration {
"Turn on ChangeManager, so delete files will go to cmrootdir."),
REPLCMDIR("hive.repl.cmrootdir","/user/${system:user.name}/cmroot/",
"Root dir for ChangeManager, used for deleted files."),
-REPLCMRETIAN("hive.repl.cm.retain","24h",
-new TimeValidator(TimeUnit.HOURS),
+REPLCMRETIAN("hive.repl.cm.retain","7d",
+new TimeValidator(TimeUnit.DAYS),
"Time to retain removed files in cmrootdir."),
REPLCMENCRYPTEDDIR("hive.repl.cm.encryptionzone.rootdir", ".cmroot",
"Root dir for ChangeManager if encryption zones are enabled, used
for deleted files."),
@@ -571,7 +571,7 @@ public class HiveConf extends Configuration {
REPL_ADD_RAW_RESERVED_NAMESPACE("hive.repl.add.raw.reserved.namespace",
false,
"For TDE with same encryption keys on source and target, allow Distcp
super user to access \n"
+ "the raw bytes from filesystem without decrypting on source and
then encrypting on target."),
-REPL_INCLUDE_EXTERNAL_TABLES("hive.repl.include.external.tables", false,
+REPL_INCLUDE_EXTERNAL_TABLES("hive.repl.include.external.tables", true,
"Indicates if repl dump should include information about external
tables. It should be \n"
+ "used in conjunction with 'hive.repl.dump.metadata.only' set to
false. if 'hive.repl.dump.metadata.only' \n"
+ " is set to true then this config parameter has no effect as
external table meta data is flushed \n"
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
index 8f1bc40..e46a520 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
@@ -1316,16 +1316,16 @@ public class TestMetaStoreMultipleEncryptionZones {
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer,
part11.getName(), fileChksum11,
ReplChangeManager.getInstance(conf).getCmRoot(part11).toString()),
-now - 86400*1000*2, now - 86400*1000*2);
+now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer,
part21.getName(), fileChksum21,
ReplChangeManager.getInstance(conf).getCmRoot(part21).toString()),
-now - 86400*1000*2, now - 86400*1000*2);
+now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer,
part31.getName(), fileChksum31,
ReplChangeManager.getInstance(conf).getCmRoot(part31).toString()),
-now - 86400*1000*2, now - 86400*1000*2);
+now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer,
part32.getName(), fileChksum32,
ReplChangeManager.getInstance(conf).getCmRoot(part32).toString()),
-now - 86400*1000*2, now - 86400*1000*2);
+now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
ReplChangeManager.scheduleCMClearer(hiveConfCmClearer);
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRep
[hive] branch master updated: HIVE-23786 HMS server side filter with Ranger (Sam An reviewed by Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 1a1fd15 HIVE-23786 HMS server side filter with Ranger (Sam An
reviewed by Peter Vary)
1a1fd15 is described below
commit 1a1fd15e3daa69e6dd5816ec877c3e8634cb7718
Author: Sam An
AuthorDate: Tue Jul 7 10:34:08 2020 -0700
HIVE-23786 HMS server side filter with Ranger (Sam An reviewed by Peter
Vary)
---
.../plugin/metastore/HiveMetaStoreAuthorizer.java | 367 ++---
.../plugin/metastore/HiveMetaStoreAuthzInfo.java | 88 ++---
.../plugin/metastore/events/ReadDatabaseEvent.java | 80 +
.../plugin/metastore/events/ReadTableEvent.java| 73
.../filtercontext/DatabaseFilterContext.java | 76 +
.../filtercontext/TableFilterContext.java | 98 ++
.../metastore/TestHiveMetaStoreAuthorizer.java | 47 +++
7 files changed, 735 insertions(+), 94 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java
index 434d1c9..09d5946 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java
@@ -18,10 +18,12 @@
*/
package org.apache.hadoop.hive.ql.security.authorization.plugin.metastore;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreFilterHook;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.MetaStorePreEventListener;
import org.apache.hadoop.hive.metastore.TableType;
@@ -34,6 +36,11 @@ import
org.apache.hadoop.hive.metastore.events.PreCreateTableEvent;
import org.apache.hadoop.hive.metastore.events.PreDropTableEvent;
import org.apache.hadoop.hive.metastore.events.PreEventContext;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
+import org.apache.hadoop.hive.metastore.api.Catalog;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionSpec;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.security.HiveMetastoreAuthenticationProvider;
@@ -45,17 +52,21 @@ import
org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionC
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactoryImpl;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
+import
org.apache.hadoop.hive.ql.security.authorization.plugin.metastore.filtercontext.DatabaseFilterContext;
+import
org.apache.hadoop.hive.ql.security.authorization.plugin.metastore.filtercontext.TableFilterContext;
import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
/**
* HiveMetaStoreAuthorizer : Do authorization checks on MetaStore Events in
MetaStorePreEventListener
*/
-public class HiveMetaStoreAuthorizer extends MetaStorePreEventListener {
- private static final LogLOG =
LogFactory.getLog(HiveMetaStoreAuthorizer.class);
+public class HiveMetaStoreAuthorizer extends MetaStorePreEventListener
implements MetaStoreFilterHook {
+ private static final Log LOG =
LogFactory.getLog(HiveMetaStoreAuthorizer.class);
private static final ThreadLocal tConfig = new
ThreadLocal() {
@Override
@@ -80,50 +91,278 @@ public class HiveMetaStoreAuthorizer extends
MetaStorePreEventListener {
}
@Override
- public final void onEvent(PreEventContext preEventContext) throws
MetaException, NoSuchObjectException, InvalidOperationException {
-if (LOG.isDebugEnabled()) {
- LOG.debug("==> HiveMetaStoreAuthorizer.onEvent(): EventType=" +
preEventContext.getEventType());
+ public final void onEvent(PreEventContext preEventContext)
+ throws MetaException, NoSuchObjectException, InvalidOperationException {
+LOG.debug("==> HiveMetaStoreAuthorizer.onEvent(): EventType=" +
preEventContext.getEventType());
+
+try {
+ HiveAuthorizer hiveAuthorizer = createHiveMetaStoreAuthorizer();
+ if (!skipAuthorization()) {
+H
[hive] branch master updated: HIVE-23953 : Use task counter information to compute keycount during hashtable loading (#1340)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ee186af HIVE-23953 : Use task counter information to compute keycount
during hashtable loading (#1340)
ee186af is described below
commit ee186af06e552bdb9790a783c3b0ff65953171e6
Author: mahesh kumar behera
AuthorDate: Tue Aug 4 16:59:39 2020 +0530
HIVE-23953 : Use task counter information to compute keycount during
hashtable loading (#1340)
HIVE-23953 : Use task counter information to compute keycount during
hashtable loading (Mahesh Kumar Behera, reviewed by Rajesh Balamohan)
---
.../hadoop/hive/ql/exec/tez/HashTableLoader.java | 29 --
.../fast/VectorMapJoinFastHashTableLoader.java | 28 +++--
2 files changed, 43 insertions(+), 14 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
index 9cb..fcb2cc5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.tez.runtime.api.Input;
import org.apache.tez.runtime.api.LogicalInput;
import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.apache.tez.runtime.api.AbstractLogicalInput;
/**
* HashTableLoader for Tez constructs the hashtable from records read from
@@ -219,7 +220,19 @@ public class HashTableLoader implements
org.apache.hadoop.hive.ql.exec.HashTable
}
isFirstKey = false;
Long keyCountObj = parentKeyCounts.get(pos);
-long keyCount = (keyCountObj == null) ? -1 : keyCountObj.longValue();
+long estKeyCount = (keyCountObj == null) ? -1 : keyCountObj;
+
+long inputRecords = -1;
+try {
+ //TODO : Need to use class instead of string.
+ // https://issues.apache.org/jira/browse/HIVE-23981
+ inputRecords = ((AbstractLogicalInput)
input).getContext().getCounters().
+ findCounter("org.apache.tez.common.counters.TaskCounter",
+ "APPROXIMATE_INPUT_RECORDS").getValue();
+} catch (Exception e) {
+ LOG.debug("Failed to get value for counter
APPROXIMATE_INPUT_RECORDS", e);
+}
+long keyCount = Math.max(estKeyCount, inputRecords);
long memory = 0;
if (useHybridGraceHashJoin) {
@@ -242,8 +255,9 @@ public class HashTableLoader implements
org.apache.hadoop.hive.ql.exec.HashTable
tableContainer = new HashMapWrapper(hconf, keyCount);
}
-LOG.info("Loading hash table for input: {} cacheKey: {}
tableContainer: {} smallTablePos: {}", inputName,
- cacheKey, tableContainer.getClass().getSimpleName(), pos);
+LOG.info("Loading hash table for input: {} cacheKey: {}
tableContainer: {} smallTablePos: {} " +
+"estKeyCount : {} keyCount : {}", inputName, cacheKey,
+tableContainer.getClass().getSimpleName(), pos, estKeyCount,
keyCount);
tableContainer.setSerde(keyCtx, valCtx);
long startTime = System.currentTimeMillis();
@@ -272,11 +286,12 @@ public class HashTableLoader implements
org.apache.hadoop.hive.ql.exec.HashTable
tableContainer.seal();
mapJoinTables[pos] = tableContainer;
if (doMemCheck) {
- LOG.info("Finished loading hash table for input: {} cacheKey: {}
numEntries: {} estimatedMemoryUsage: {}",
-inputName, cacheKey, numEntries,
tableContainer.getEstimatedMemorySize());
+ LOG.info("Finished loading hash table for input: {} cacheKey: {}
numEntries: {} " +
+ "estimatedMemoryUsage: {} Load Time : {} ",
+inputName, cacheKey, numEntries,
tableContainer.getEstimatedMemorySize(), delta);
} else {
- LOG.info("Finished loading hash table for input: {} cacheKey: {}
numEntries: {}", inputName, cacheKey,
-numEntries);
+ LOG.info("Finished loading hash table for input: {} cacheKey: {}
numEntries: {} Load Time : {} ",
+ inputName, cacheKey, numEntries, delta);
}
} catch (Exception e) {
throw new HiveException(e);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
index a011027..0eba50c 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.io
[hive] branch master updated: HIVE-23763: Query based minor compaction produces wrong files when ro… (#1327)
This is an automated email from the ASF dual-hosted git repository.
kuczoram pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 98c925c HIVE-23763: Query based minor compaction produces wrong files
when ro… (#1327)
98c925c is described below
commit 98c925c8cb52e410c9aa02a499d2ac15a6d6777b
Author: kuczoram
AuthorDate: Tue Aug 4 12:35:52 2020 +0200
HIVE-23763: Query based minor compaction produces wrong files when ro…
(#1327)
* HIVE-23763: Query based minor compaction produces wrong files when rows
with different buckets Ids are processed by the same FileSinkOperator
* HIVE-23763: Fix the MM compaction tests
* HIVE-23763: Address the review comments
---
.../hive/ql/txn/compactor/CompactorOnTezTest.java | 150 ++-
.../hive/ql/txn/compactor/CompactorTestUtil.java | 13 +-
.../ql/txn/compactor/TestCrudCompactorOnTez.java | 290 -
.../ql/txn/compactor/TestMmCompactorOnTez.java | 18 +-
.../hadoop/hive/ql/exec/FileSinkOperator.java | 64 +++--
.../hadoop/hive/ql/exec/ReduceSinkOperator.java| 14 +
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 21 +-
.../opconventer/HiveOpConverterUtils.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 25 +-
.../org/apache/hadoop/hive/ql/plan/PlanUtils.java | 3 +-
.../apache/hadoop/hive/ql/plan/ReduceSinkDesc.java | 9 +
.../hive/ql/txn/compactor/QueryCompactor.java | 12 +
12 files changed, 551 insertions(+), 70 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorOnTezTest.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorOnTezTest.java
index 71232de..08b9039 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorOnTezTest.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorOnTezTest.java
@@ -35,6 +35,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import static
org.apache.hadoop.hive.ql.txn.compactor.CompactorTestUtil.executeStatementOnDriverAndReturnResults;
import static
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.executeStatementOnDriver;
@@ -53,6 +55,7 @@ public class CompactorOnTezTest {
protected IMetaStoreClient msClient;
protected IDriver driver;
protected boolean runsOnTez = true;
+ protected boolean mmCompaction = false;
@Before
// Note: we create a new conf and driver object before every test
@@ -95,6 +98,17 @@ public class CompactorOnTezTest {
conf.set("hive.tez.container.size", "128");
conf.setBoolean("hive.merge.tezfiles", false);
conf.setBoolean("hive.in.tez.test", true);
+if (!mmCompaction) {
+ // We need these settings to create a table which is not bucketed, but
contains multiple files.
+ // If these parameters are set when inserting 100 rows into the table,
the rows will
+ // be distributed into multiple files. This setup is used in the
testMinorCompactionWithoutBuckets,
+ // testMinorCompactionWithoutBucketsInsertOverwrite and
testMajorCompactionWithoutBucketsInsertAndDeleteInsertOverwrite
+ // tests in the TestCrudCompactorOnTez class.
+ // This use case has to be tested because of HIVE-23763. The MM
compaction is not affected by this issue,
+ // therefore no need to set these configs for MM compaction.
+ conf.set("tez.grouping.max-size", "1024");
+ conf.set("tez.grouping.min-size", "1");
+}
}
@After public void tearDown() {
@@ -218,6 +232,77 @@ public class CompactorOnTezTest {
}
/**
+ * This method is for creating a non-bucketed table in which the data is
distributed
+ * into multiple splits. The initial data is 100 rows and it should be
split into
+ * multiple files, like bucket_01, bucket_02, ...
+ * This is needed because the MINOR compactions had issues with tables
like this. (HIVE-23763)
+ * @param dbName
+ * @param tblName
+ * @param tempTblName
+ * @param createDeletes
+ * @param createInserts
+ * @param insertOverwrite
+ * @throws Exception
+ */
+void createTableWithoutBucketWithMultipleSplits(String dbName, String
tblName, String tempTblName,
+boolean createDeletes, boolean createInserts, boolean insertOverwrite)
throws Exception {
+ if (dbName != null) {
+tblName = dbName + "." + tblName;
+tempTblName = dbName + "." + tempTblName;
+ }
+
+ executeStatementOnDriver("drop table if exists " + tblName, driver);
+ StringBuilder query = new StringBuilder();
+ query.append("create table ").append(tblName).append(" (a string, b
string, c st
[hive] branch master updated: HIVE-23929: Run compaction as partition owner (Laszlo Pinter reviewed by Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 3d85d8b HIVE-23929: Run compaction as partition owner (Laszlo Pinter
reviewed by Peter Vary)
3d85d8b is described below
commit 3d85d8bbcf473aea3e3b5a00db662a5d5ea973d7
Author: László Pintér <4102+laszlopinte...@users.noreply.github.com>
AuthorDate: Tue Aug 4 12:17:18 2020 +0200
HIVE-23929: Run compaction as partition owner (Laszlo Pinter reviewed by
Peter Vary)
Closes (#1309)
---
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index 784c95b..91898ee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -520,7 +520,7 @@ public class Worker extends RemoteCompactorThread
implements MetaStoreThread {
if (runJobAsSelf(ci.runAs)) {
mr.run(conf, jobName.toString(), t, p, sd, tblValidWriteIds, ci, su,
msc, dir);
} else {
- UserGroupInformation ugi =
UserGroupInformation.createProxyUser(t.getOwner(),
+ UserGroupInformation ugi =
UserGroupInformation.createProxyUser(ci.runAs,
UserGroupInformation.getLoginUser());
final Partition fp = p;
final CompactionInfo fci = ci;
[hive] branch master updated: disable unstable tests
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new fdfd136 disable unstable tests
fdfd136 is described below
commit fdfd136083de39493ea36f64ce34ca22c348c537
Author: Zoltan Haindrich
AuthorDate: Tue Aug 4 09:57:42 2020 +
disable unstable tests
---
.../hcatalog/templeton/TestConcurrentJobRequestsThreadsAndTimeout.java | 1 +
.../hadoop/hive/ql/parse/TestStatsReplicationScenariosMigration.java | 1 +
.../ql/parse/TestStatsReplicationScenariosMigrationNoAutogather.java | 1 +
ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q | 3 ++-
ql/src/test/queries/clientpositive/druidmini_expressions.q | 1 +
ql/src/test/queries/clientpositive/kafka_storage_handler.q | 2 ++
6 files changed, 8 insertions(+), 1 deletion(-)
diff --git
a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestConcurrentJobRequestsThreadsAndTimeout.java
b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestConcurrentJobRequestsThreadsAndTimeout.java
index d351a98..abb81a0 100644
---
a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestConcurrentJobRequestsThreadsAndTimeout.java
+++
b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestConcurrentJobRequestsThreadsAndTimeout.java
@@ -33,6 +33,7 @@ import static org.junit.Assert.assertTrue;
* Requests and job request execution time outs. Verify that we get
appropriate exceptions
* and exception message.
*/
+@org.junit.Ignore("HIVE-23983")
public class TestConcurrentJobRequestsThreadsAndTimeout extends
ConcurrentJobRequestsTestBase {
private static AppConfig config;
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMigration.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMigration.java
index 4b3bf4f..8544d32 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMigration.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMigration.java
@@ -29,6 +29,7 @@ import java.util.Map;
/**
* Tests statistics replication for ACID tables.
*/
+@org.junit.Ignore("HIVE-23986")
public class TestStatsReplicationScenariosMigration extends
TestStatsReplicationScenarios {
@Rule
public final TestName testName = new TestName();
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMigrationNoAutogather.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMigrationNoAutogather.java
index 7fc3bc5..21790c5 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMigrationNoAutogather.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMigrationNoAutogather.java
@@ -29,6 +29,7 @@ import java.util.Map;
/**
* Tests statistics replication for ACID tables.
*/
+@org.junit.Ignore("HIVE-23982")
public class TestStatsReplicationScenariosMigrationNoAutogather extends
TestStatsReplicationScenarios {
@Rule
public final TestName testName = new TestName();
diff --git a/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q
b/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q
index 36a68d0..eacdcfe 100644
--- a/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q
+++ b/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q
@@ -1,3 +1,4 @@
+--! qt:disabled:HIVE-23984
--! qt:dataset:alltypesorc
SET hive.vectorized.execution.enabled=true ;
SET hive.ctas.external.tables=true;
@@ -186,4 +187,4 @@ SELECT sum(cint), sum(cbigint) FROM
druid_max_size_partition ;
DROP TABLE druid_partitioned_table_0;
DROP TABLE druid_partitioned_table;
-DROP TABLE druid_max_size_partition;
\ No newline at end of file
+DROP TABLE druid_max_size_partition;
diff --git a/ql/src/test/queries/clientpositive/druidmini_expressions.q
b/ql/src/test/queries/clientpositive/druidmini_expressions.q
index 59ffb5b..2347d62 100644
--- a/ql/src/test/queries/clientpositive/druidmini_expressions.q
+++ b/ql/src/test/queries/clientpositive/druidmini_expressions.q
@@ -1,3 +1,4 @@
+--! qt:disabled:HIVE-23984
--! qt:dataset:druid_table_alltypesorc
SET hive.ctas.external.tables=true;
diff --git a/ql/src/test/queries/clientpositive/kafka_storage_handler.q
b/ql/src/test/queries/clientpositive/kafka_storage_handler.q
index f4670f5..b8283e7 100644
--- a/ql/src/test/queries/clientpositive/kafka_storage_handler.q
+++ b/ql/src/test/queries/clientpositive/kafka_storage_handler.q
@@ -1,3 +1,5 @@
+--! qt:disabled:HIVE-23985
+
SET hive.vector
[hive] branch master updated: HIVE-20441: NPE in GenericUDF when hive.allow.udf.load.on.demand is set to true (Zhihua Deng reviewed by Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 34ed983 HIVE-20441: NPE in GenericUDF when
hive.allow.udf.load.on.demand is set to true (Zhihua Deng reviewed by Peter
Vary)
34ed983 is described below
commit 34ed9834abaf06537a127cbf6f12560ea7a4418a
Author: dengzh
AuthorDate: Tue Aug 4 15:35:33 2020 +0800
HIVE-20441: NPE in GenericUDF when hive.allow.udf.load.on.demand is set to
true (Zhihua Deng reviewed by Peter Vary)
Closes (#1242)
---
.../org/apache/hadoop/hive/ql/exec/Registry.java | 5 +-
.../parse/type/TestExprProcessorGetFuncExpr.java | 81 ++
2 files changed, 84 insertions(+), 2 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
index 6ceea2f..1f6ae08 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
@@ -293,9 +293,10 @@ public class Registry {
if (registerToSession) {
String qualifiedName = FunctionUtils.qualifyFunctionName(
functionName, SessionState.get().getCurrentDatabase().toLowerCase());
- if (registerToSessionRegistry(qualifiedName, function) != null) {
+ FunctionInfo newFunction = registerToSessionRegistry(qualifiedName,
function);
+ if (newFunction != null) {
addFunction(functionName, function);
-return function;
+return newFunction;
}
} else {
addFunction(functionName, function);
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestExprProcessorGetFuncExpr.java
b/ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestExprProcessorGetFuncExpr.java
new file mode 100644
index 000..f944f91
--- /dev/null
+++
b/ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestExprProcessorGetFuncExpr.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse.type;
+
+import org.antlr.runtime.CommonToken;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.FunctionType;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Check the genericUDF field of FunctionInfo returned on demand.
+ *
+ */
+public class TestExprProcessorGetFuncExpr {
+
+ @Before
+ public void setUp() throws Exception {
+HiveConf hiveConf = new HiveConf();
+hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_ALLOW_UDF_LOAD_ON_DEMAND, true);
+SessionState sessionState = new SessionState(hiveConf,
System.getProperty("user.name"));
+SessionState.setCurrentSessionState(sessionState);
+Function function = new Function("myupper",
sessionState.getCurrentDatabase(),
+"org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper",
sessionState.getUserName(),
+PrincipalType.USER, (int) (System.currentTimeMillis() / 1000),
FunctionType.JAVA, null);
+Hive.get().createFunction(function);
+ }
+
+ @Test
+ public void testLookupFunctionOnDemand() throws Exception {
+TypeCheckProcFactory.DefaultExprProcessor defaultExprProcessor =
+ExprNodeTypeCheck.getExprNodeDefaultExprProcessor();
+ASTNode funcExpr = new ASTNode(new CommonToken(HiveParser.TOK_FUNCTION,
"TOK_FUNCTION"));
+funcExpr.addChild(new ASTNode(new CommonToken(HiveParser.Identifier,
"myupper")));
+funcExpr.ad
[hive] branch master updated (ad28ea0 -> 6d797bf)
This is an automated email from the ASF dual-hosted git repository. pvary pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from ad28ea0 HIVE-23949: Introduce caching layer in HS2 to accelerate query compilation (Soumyakanti Das, reviewed by Vineet Garg, Jesus Camacho Rodriguez) (addendum) add 6d797bf HIVE-23819: Use ranges in ValidReadTxnList serialization (Peter Varga reviewed by Peter Vary) No new revisions were added by this update. Summary of changes: .../hadoop/hive/common/ValidReadTxnList.java | 139 +++-- .../hadoop/hive/common/TestValidReadTxnList.java | 16 +++ 2 files changed, 117 insertions(+), 38 deletions(-)
