[hive] branch master updated: HIVE-25403: Fix from_unixtime() to consider leap seconds (Sruthi Mooriyathvariam, reviewed by Ashish Sharma, Adesh Rao, Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 103ac9a HIVE-25403: Fix from_unixtime() to consider leap seconds
(Sruthi Mooriyathvariam, reviewed by Ashish Sharma, Adesh Rao, Sankar Hariappan)
103ac9a is described below
commit 103ac9ab352d15e63885f5b62d63f55011dcc01c
Author: Sruthi Mooriyathvariam
AuthorDate: Wed Aug 11 11:26:36 2021 +0530
HIVE-25403: Fix from_unixtime() to consider leap seconds (Sruthi
Mooriyathvariam, reviewed by Ashish Sharma, Adesh Rao, Sankar Hariappan)
Signed-off-by: Sankar Hariappan
Closes (#2550)
---
.../ql/udf/generic/GenericUDFFromUnixTime.java | 96 --
.../ql/udf/generic/TestGenericUDFFromUnixTime.java | 137
ql/src/test/queries/clientpositive/foldts.q| 8 +-
ql/src/test/queries/clientpositive/udf5.q | 29 +
.../test/results/clientpositive/llap/foldts.q.out | 27
ql/src/test/results/clientpositive/llap/udf5.q.out | 142 +
6 files changed, 370 insertions(+), 69 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUnixTime.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUnixTime.java
index 66418ac..fb634bc 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUnixTime.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUnixTime.java
@@ -18,28 +18,26 @@
package org.apache.hadoop.hive.ql.udf.generic;
-import java.text.SimpleDateFormat;
+import java.time.Instant;
import java.time.ZoneId;
-import java.util.Date;
-import java.util.TimeZone;
-import org.apache.commons.lang3.StringUtils;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+
import org.apache.hadoop.hive.common.type.TimestampTZUtil;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.MapredContext;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
import
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.Text;
+import static
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP;
/**
* GenericUDFFromUnixTime.
@@ -54,28 +52,19 @@ public class GenericUDFFromUnixTime extends GenericUDF {
private transient IntObjectInspector inputIntOI;
private transient LongObjectInspector inputLongOI;
- private transient Converter inputTextConverter;
private transient ZoneId timeZone;
private transient final Text result = new Text();
-
- private transient SimpleDateFormat formatter = new
SimpleDateFormat("-MM-dd HH:mm:ss");
- private transient String lastFormat = null;
-
+ private transient String lastFormat ="-MM-dd HH:mm:ss";
+ private transient DateTimeFormatter FORMATTER =
DateTimeFormatter.ofPattern(lastFormat);
+ private transient Converter[] converters = new Converter[2];
+ private transient PrimitiveObjectInspector.PrimitiveCategory[] inputTypes =
new PrimitiveObjectInspector.PrimitiveCategory[2];
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws
UDFArgumentException {
-if (arguments.length < 1) {
- throw new UDFArgumentLengthException("The function " +
getName().toUpperCase() +
- "requires at least one argument");
-}
-if (arguments.length > 2) {
- throw new UDFArgumentLengthException("Too many arguments for the
function " + getName().toUpperCase());
-}
-for (ObjectInspector argument : arguments) {
- if (argument.getCategory() != Category.PRIMITIVE) {
-throw new UDFArgumentException(getName().toUpperCase() +
-" only takes primitive types, got " + argument.getTypeName());
- }
+checkArgsSize(arguments, 1, 2);
+
+for (int i = 0; i < arguments.length; i++) {
+ checkArgPrimitive(arguments, i);
}
PrimitiveObjectInspector arg0OI =
[hive] branch master updated (9b5feec -> a903002)
This is an automated email from the ASF dual-hosted git repository. ychena pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 9b5feec HIVE-25410: CommonMergeJoin fails for ARRAY join keys with varying sizes (okumin reviewed by Stamatis Zampetakis) add a903002 HIVE-25423: Add new test driver to automatically launch and load external database (Same as PR#2559)(#2580) (Dantong Dong, reviewed by Sai Hemanth Gantasala and Yongzhi Chen) No new revisions were added by this update. Summary of changes: data/scripts/derby/q_test_extDB_cleanup.derby.sql | 3 + data/scripts/derby/q_test_extDB_init.derby.sql | 27 ++ data/scripts/mysql/q_test_extDB_cleanup.mysql.sql | 3 + data/scripts/mysql/q_test_extDB_init.mysql.sql | 23 ++ .../postgres/q_test_extDB_cleanup.postgres.sql | 3 + .../postgres/q_test_extDB_init.postgres.sql| 23 ++ itests/qtest/pom.xml | 18 ++ .../hive/cli/TestMiniLlapExtDBCliDriver.java | 65 + .../test/resources/testconfiguration.properties| 5 + .../hadoop/hive/cli/control/AbstractCliConfig.java | 5 + .../apache/hadoop/hive/cli/control/CliConfigs.java | 24 ++ .../hadoop/hive/cli/control/CoreCliDriver.java | 5 + .../org/apache/hadoop/hive/ql/QTestArguments.java | 17 ++ .../org/apache/hadoop/hive/ql/QTestExternalDB.java | 57 .../java/org/apache/hadoop/hive/ql/QTestUtil.java | 62 - .../hive/ql/externalDB/AbstractExternalDB.java | 288 + .../hadoop/hive/ql/externalDB/MySQLExternalDB.java | 59 + .../hive/ql/externalDB/PostgresExternalDB.java | 67 + .../queries/clientpositive/dataconnector_mysql.q | 19 ++ .../clientpositive/llap/dataconnector_mysql.q.out | 68 + .../dataconnector/jdbc/MySQLConnectorProvider.java | 2 +- .../ptest2/conf/deployed/master-mr2.properties | 9 +- 22 files changed, 849 insertions(+), 3 deletions(-) create mode 100644 data/scripts/derby/q_test_extDB_cleanup.derby.sql create mode 100644 data/scripts/derby/q_test_extDB_init.derby.sql create mode 100644 data/scripts/mysql/q_test_extDB_cleanup.mysql.sql create mode 100644 data/scripts/mysql/q_test_extDB_init.mysql.sql create mode 100644 data/scripts/postgres/q_test_extDB_cleanup.postgres.sql create mode 100644 data/scripts/postgres/q_test_extDB_init.postgres.sql create mode 100644 itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapExtDBCliDriver.java create mode 100644 itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestExternalDB.java create mode 100644 itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java create mode 100644 itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MySQLExternalDB.java create mode 100644 itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/PostgresExternalDB.java create mode 100644 ql/src/test/queries/clientpositive/dataconnector_mysql.q create mode 100644 ql/src/test/results/clientpositive/llap/dataconnector_mysql.q.out
[hive] branch master updated: HIVE-25410: CommonMergeJoin fails for ARRAY join keys with varying sizes (okumin reviewed by Stamatis Zampetakis)
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9b5feec HIVE-25410: CommonMergeJoin fails for ARRAY join keys with
varying sizes (okumin reviewed by Stamatis Zampetakis)
9b5feec is described below
commit 9b5feec4dc475d6ee38c6d11130fb19d351a718d
Author: okumin
AuthorDate: Sat Jul 31 01:31:11 2021 +0900
HIVE-25410: CommonMergeJoin fails for ARRAY join keys with varying sizes
(okumin reviewed by Stamatis Zampetakis)
CommonMergeJoinOperator fails with ArrayIndexOutOfBoundsException when
the column contains ARRAYs of different size. Before this change the
comparators were created only once for the first comparison and they
were reused afterwards causing exception when subsequent ARRAY elements
were bigger.
STRUCT are not affected since the number of elements is consistent
across records.
Closes #2551
---
.../hadoop/hive/ql/exec/HiveStructComparator.java | 19 ++-
.../hive/ql/exec/WritableComparatorFactory.java| 3 +-
.../clientpositive/smb_mapjoin_complex_type.q | 4 +-
.../clientpositive/test_join_complex_type.q| 8 +-
.../llap/test_join_complex_type.q.out | 186 +++--
.../clientpositive/smb_mapjoin_complex_type.q.out | 19 ++-
6 files changed, 162 insertions(+), 77 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/HiveStructComparator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/HiveStructComparator.java
index 50d02cc..d8abf72 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/HiveStructComparator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/HiveStructComparator.java
@@ -19,12 +19,15 @@ package org.apache.hadoop.hive.ql.exec;
import org.apache.hadoop.hive.ql.util.NullOrdering;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
+import java.util.ArrayList;
import java.util.List;
+/**
+ * A WritableComparator to compare STRUCT or ARRAY objects.
+ */
final class HiveStructComparator extends HiveWritableComparator {
-private WritableComparator[] comparator = null;
+private final List comparators = new ArrayList<>();
HiveStructComparator(boolean nullSafe, NullOrdering nullOrdering) {
super(nullSafe, nullOrdering);
@@ -45,16 +48,14 @@ final class HiveStructComparator extends
HiveWritableComparator {
if (a1.size() == 0) {
return 0;
}
-if (comparator == null) {
-comparator = new WritableComparator[a1.size()];
-// For struct all elements may not be of same type, so create
comparator for each entry.
-for (int i = 0; i < a1.size(); i++) {
-comparator[i] = WritableComparatorFactory.get(a1.get(i),
nullSafe, nullOrdering);
-}
+// For array, the length may not be fixed, so extend comparators on
demand
+for (int i = comparators.size(); i < a1.size(); i++) {
+// For struct, all elements may not be of same type, so create
comparator for each entry.
+comparators.add(i, WritableComparatorFactory.get(a1.get(i),
nullSafe, nullOrdering));
}
result = 0;
for (int i = 0; i < a1.size(); i++) {
-result = comparator[i].compare(a1.get(i), a2.get(i));
+result = comparators.get(i).compare(a1.get(i), a2.get(i));
if (result != 0) {
return result;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/WritableComparatorFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/WritableComparatorFactory.java
index 17ae06d..ff9ada4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/WritableComparatorFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/WritableComparatorFactory.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.exec;
import org.apache.hadoop.hive.ql.util.NullOrdering;
import
org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import java.util.List;
import java.util.Map;
@@ -27,7 +26,7 @@ import java.util.Map;
public final class WritableComparatorFactory {
public static WritableComparator get(Object key, boolean nullSafe,
NullOrdering nullOrdering) {
if (key instanceof List) {
-// For array type struct is used as we do not know if all elements
of array are of same type.
+// STRUCT or ARRAY are expressed as java.util.List
return new HiveStructComparator(nullSafe, nullOrdering);
} else if (key instanceof Map) {
// TODO : https://issues.apache.org/jira/browse/HIVE-25042
diff --git
[hive] branch master updated (8785dd2 -> 0bc7d1b)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 8785dd2 HIVE-25350. Replication fails for external tables on setting owner/groups. (#2498)(Ayush Saxena, reviewed by Arko Sharma) add 0bc7d1b HIVE-25417 : Null bit vector is not handled while getting the stats for Postgres backend (Mahesh Kumar Behera, reviewed by Zoltan Haindrich,Ramesh Kumar Thangarajan) No new revisions were added by this update. Summary of changes: .../queries/clientpositive/nullBitVectorStats.q| 210 .../clientpositive/llap/nullBitVectorStats.q.out | 579 + .../apache/hadoop/hive/metastore/ObjectStore.java | 10 - .../hadoop/hive/metastore/StatObjectConverter.java | 22 +- .../model/MPartitionColumnStatistics.java | 25 +- .../metastore/model/MTableColumnStatistics.java| 25 +- 6 files changed, 839 insertions(+), 32 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/nullBitVectorStats.q create mode 100644 ql/src/test/results/clientpositive/llap/nullBitVectorStats.q.out
