[hive] branch master updated: HIVE-27474: Deprecate registerUDF with function resources as param for in Registry (#4458). (Suprith Chandrashekharachar, reviewed by Ayush Saxena)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 24a82a65f96 HIVE-27474: Deprecate registerUDF with function resources
as param for in Registry (#4458). (Suprith Chandrashekharachar, reviewed by
Ayush Saxena)
24a82a65f96 is described below
commit 24a82a65f96b65eeebe4e23b2fec425037a70216
Author: Suprith Kumar Suvarneshwar
AuthorDate: Mon Jul 10 23:32:47 2023 -0700
HIVE-27474: Deprecate registerUDF with function resources as param for in
Registry (#4458). (Suprith Chandrashekharachar, reviewed by Ayush Saxena)
---
.../org/apache/hadoop/hive/ql/exec/Registry.java| 21 +
1 file changed, 21 insertions(+)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
index d70cccd297a..d01da9f8f37 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
@@ -146,11 +146,21 @@ public class Registry {
}
+ /**
+ * @deprecated From next release, replaced by {@link #registerUDF(String,
Class, boolean)}
+ * Deprecated because method unnecessarily accepts and passes
FunctionResource vararg param
+ */
+ @Deprecated
public FunctionInfo registerUDF(String functionName,
Class UDFClass, boolean isOperator, FunctionResource...
resources) {
return registerUDF(functionName, UDFClass, isOperator,
functionName.toLowerCase(), resources);
}
+ /**
+ * @deprecated From next release, replaced by {@link #registerUDF(String,
Class, boolean, String)}
+ * Deprecated because method unnecessarily accepts FunctionResource vararg
param
+ */
+ @Deprecated
public FunctionInfo registerUDF(String functionName,
Class UDFClass, boolean isOperator, String displayName,
FunctionResource... resources) {
@@ -158,6 +168,17 @@ public class Registry {
return registerUDF(functionName, functionType, UDFClass, isOperator,
displayName);
}
+ public FunctionInfo registerUDF(String functionName,
+ Class UDFClass, boolean
isOperator) {
+return registerUDF(functionName, UDFClass, isOperator,
functionName.toLowerCase());
+ }
+
+ public FunctionInfo registerUDF(String functionName,
+ Class UDFClass, boolean
isOperator, String displayName) {
+ FunctionType functionType = isNative ? FunctionType.BUILTIN :
FunctionType.TEMPORARY;
+ return registerUDF(functionName, functionType, UDFClass, isOperator,
displayName);
+ }
+
private FunctionInfo registerUDF(String functionName, FunctionType
functionType,
Class UDFClass, boolean isOperator, String displayName,
FunctionResource... resources) {
[hive] branch master updated: HIVE-27438: Audit leader election event failed in non-appendable filesystems (#4416) (Zhihua Deng, reviewed by Sai Hemanth Gantasala)
This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8fcef2c9740 HIVE-27438: Audit leader election event failed in
non-appendable filesystems (#4416) (Zhihua Deng, reviewed by Sai Hemanth
Gantasala)
8fcef2c9740 is described below
commit 8fcef2c97404a6226071a26a8b637bc019db4f45
Author: dengzh
AuthorDate: Tue Jul 11 14:03:50 2023 +0800
HIVE-27438: Audit leader election event failed in non-appendable
filesystems (#4416) (Zhihua Deng, reviewed by Sai Hemanth Gantasala)
---
.../hadoop/hive/metastore/conf/MetastoreConf.java | 15 +++-
.../hadoop/hive/metastore/HiveMetaStore.java | 17 ++--
.../hive/metastore/leader/AuditLeaderListener.java | 72 +---
.../hive/metastore/leader/CompactorTasks.java | 1 +
.../hive/metastore/leader/HouseKeepingTasks.java | 18 +++-
.../metastore/leader/LeaderElectionContext.java| 3 +-
.../metastore/leader/LeaderElectionFactory.java| 2 +-
.../hive/metastore/leader/LeaseLeaderElection.java | 22 ++---
...aderElection.java => StaticLeaderElection.java} | 10 +--
.../hive/metastore/leader/StatsUpdaterTask.java| 1 +
.../hive/metastore/leader/TestLeaderElection.java | 2 +-
.../hive/metastore/leader/TestLeaderListener.java | 98 ++
12 files changed, 210 insertions(+), 51 deletions(-)
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index c1354341c34..4e0c136fb5d 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -568,11 +568,20 @@ public class MetastoreConf {
METASTORE_HOUSEKEEPING_LEADER_ELECTION("metastore.housekeeping.leader.election",
"metastore.housekeeping.leader.election",
"host", new StringSetValidator("host", "lock"),
-"If sets to host, HMS will choose the leader by the configured
metastore.housekeeping.leader.hostname.\n" +
-"When configured to lock, HMS will use hive locks to elect the
leader."),
+"Set to host, HMS will choose the leader by the configured
metastore.housekeeping.leader.hostname.\n" +
+"Set to lock, HMS will use the hive lock to elect the leader."),
METASTORE_HOUSEKEEPING_LEADER_AUDITTABLE("metastore.housekeeping.leader.auditTable",
"metastore.housekeeping.leader.auditTable", "",
-"Audit the Metastore leader changes to the target table when
configured."),
+"Audit the leader election event to a plain json table when
configured."),
+
METASTORE_HOUSEKEEPING_LEADER_NEW_AUDIT_FILE("metastore.housekeeping.leader.newAuditFile",
+"metastore.housekeeping.leader.newAuditFile", false,
+"Whether to create a new audit file in response to the new election
event " +
+"when the metastore.housekeeping.leader.auditTable is not empty.\n" +
+"True for creating a new file, false otherwise."),
+
METASTORE_HOUSEKEEPING_LEADER_AUDIT_FILE_LIMIT("metastore.housekeeping.leader.auditFiles.limit",
+"metastore.housekeeping.leader.auditFiles.limit", 10,
+"Limit the number of small audit files when
metastore.housekeeping.leader.newAuditFile is true.\n" +
+"If the number of audit files exceeds the limit, then the oldest will
be deleted."),
METASTORE_HOUSEKEEPING_THREADS_ON("metastore.housekeeping.threads.on",
"hive.metastore.housekeeping.threads.on", false,
"Whether to run the tasks under metastore.task.threads.remote on this
metastore instance or not.\n" +
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 4e49b76f6f9..f6980e2fac6 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.thrift.TProcessor;
@@ -826,28 +825,24 @@ public class HiveMetaStore extends ThriftHiveMetastore {
LeaderElectionContext context = new
Leader
[hive] branch master updated: HIVE-27472: Iceberg: Disallow creation of temporary tables (Denys Kuzmenko, reviewed by Attila Turoczy, Ayush Saxena, Simhadri Govindappa)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e489cf3a199 HIVE-27472: Iceberg: Disallow creation of temporary tables
(Denys Kuzmenko, reviewed by Attila Turoczy, Ayush Saxena, Simhadri Govindappa)
e489cf3a199 is described below
commit e489cf3a19940b843baad6999f81f7ad10f817da
Author: Denys Kuzmenko
AuthorDate: Tue Jul 11 08:31:49 2023 +0300
HIVE-27472: Iceberg: Disallow creation of temporary tables (Denys Kuzmenko,
reviewed by Attila Turoczy, Ayush Saxena, Simhadri Govindappa)
Closes #4456
---
.../org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java | 5 -
.../mr/hive/TestHiveIcebergStorageHandlerNoScan.java| 17 -
2 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index d946531d58f..605f07728d3 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -165,6 +165,9 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
@Override
public void preCreateTable(org.apache.hadoop.hive.metastore.api.Table
hmsTable) {
+if (hmsTable.isTemporary()) {
+ throw new UnsupportedOperationException("Creation of temporary iceberg
tables is not supported.");
+}
this.catalogProperties = getCatalogProperties(hmsTable);
// Set the table type even for non HiveCatalog based tables
@@ -173,7 +176,7 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
if (!Catalogs.hiveCatalog(conf, catalogProperties)) {
if
(Boolean.parseBoolean(this.catalogProperties.getProperty(hive_metastoreConstants.TABLE_IS_CTLT)))
{
-throw new RuntimeException("CTLT target table must be a HiveCatalog
table.");
+throw new UnsupportedOperationException("CTLT target table must be a
HiveCatalog table.");
}
// For non-HiveCatalog tables too, we should set the input and output
format
// so that the table can be read by other engines like Impala
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
index aedcfdff5cc..21fb270f366 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
@@ -1693,7 +1693,22 @@ public class TestHiveIcebergStorageHandlerNoScan {
testTables.propertiesForCreateTableSQL(ImmutableMap.of(;
})
.isInstanceOf(IllegalArgumentException.class)
-.hasMessageContaining(" CTLT target table must be a HiveCatalog
table");
+.hasMessageContaining("CTLT target table must be a HiveCatalog table");
+ }
+
+ @Test
+ public void testCreateTemporaryTable() {
+TableIdentifier identifier = TableIdentifier.of("default", "customers");
+String query = String.format("CREATE temporary TABLE customers
(customer_id BIGINT, first_name STRING, last_name " +
+"STRING) STORED BY iceberg %s %s",
+testTables.locationForCreateTableSQL(identifier),
+testTables.propertiesForCreateTableSQL(ImmutableMap.of()));
+
+Assertions.assertThatThrownBy(() -> {
+ shell.executeStatement(query);
+})
+.isInstanceOf(IllegalArgumentException.class)
+.hasMessageContaining("Creation of temporary iceberg tables is not
supported");
}
@Test
[hive] branch branch-3 updated: HIVE-27255: Backport of HIVE-18786: NPE in Hive windowing functions (#4472)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 720a0f271ee HIVE-27255: Backport of HIVE-18786: NPE in Hive windowing
functions (#4472)
720a0f271ee is described below
commit 720a0f271eee924d2d5c3fbd325bf38ce98fe948
Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com>
AuthorDate: Tue Jul 11 08:46:16 2023 +0530
HIVE-27255: Backport of HIVE-18786: NPE in Hive windowing functions (#4472)
* HIVE-18786: NPE in Hive windowing functions (Dongwook Kwon via Ashutosh
Chauhan)
Signed-off-by: Sankar Hariappan
Closes (#4472)
---
.../hive/ql/udf/generic/GenericUDAFEvaluator.java | 1 +
.../ql/udf/generic/TestGenericUDAFEvaluator.java | 79 ++
2 files changed, 80 insertions(+)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java
index 09e25833632..960d8fdb894 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java
@@ -149,6 +149,7 @@ public abstract class GenericUDAFEvaluator implements
Closeable {
// This function should be overriden in every sub class
// And the sub class should call super.init(m, parameters) to get mode set.
mode = m;
+partitionEvaluator = null;
return null;
}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDAFEvaluator.java
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDAFEvaluator.java
new file mode 100644
index 000..878733155ed
--- /dev/null
+++
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDAFEvaluator.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.hadoop.hive.ql.exec.PTFPartition;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
+import org.apache.hadoop.hive.ql.udf.ptf.BasePartitionEvaluator;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Answers;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.util.Collections;
+import java.util.List;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestGenericUDAFEvaluator {
+
+ @Mock(answer = Answers.CALLS_REAL_METHODS)
+ private GenericUDAFEvaluator udafEvaluator;
+
+ @Mock
+ private WindowFrameDef winFrame;
+
+ @Mock
+ private PTFPartition partition1;
+
+ @Mock
+ private ObjectInspector outputOI;
+
+ private List parameters = Collections.emptyList();
+
+ @Test
+ public void testGetPartitionWindowingEvaluatorWithoutInitCall() {
+BasePartitionEvaluator partition1Evaluator1 =
udafEvaluator.getPartitionWindowingEvaluator(
+winFrame, partition1, parameters, outputOI, false);
+
+BasePartitionEvaluator partition1Evaluator2 =
udafEvaluator.getPartitionWindowingEvaluator(
+winFrame, partition1, parameters, outputOI, false);
+
+Assert.assertEquals(partition1Evaluator1, partition1Evaluator2);
+ }
+
+ @Test
+ public void testGetPartitionWindowingEvaluatorWithInitCall() throws
HiveException {
+BasePartitionEvaluator partition1Evaluator1 =
udafEvaluator.getPartitionWindowingEvaluator(
+winFrame, partition1, parameters, outputOI, false);
+
+udafEvaluator.init(GenericUDAFEvaluator.Mode.COMPLETE, null);
+
+BasePartitionEvaluator newPartitionEvaluator =
udafEvaluator.getPartitionWindowingEvaluator(
+winFrame, partition1, parameters, outputOI, false);
+
+Assert.assertNotEquals(partition1Evaluator1, newPartitionEvaluator);
+ }
+
+}
[hive] branch dependabot/maven/org.xerial.snappy-snappy-java-1.1.10.1 deleted (was 4e391d11cd0)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/org.xerial.snappy-snappy-java-1.1.10.1 in repository https://gitbox.apache.org/repos/asf/hive.git was 4e391d11cd0 Bump snappy-java from 1.1.8.4 to 1.1.10.1 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
