http://git-wip-us.apache.org/repos/asf/hive/blob/ae008b79/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java ---------------------------------------------------------------------- diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java index a3fd7b5..3c2797e 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java @@ -18,15 +18,15 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; -import java.sql.Date; -import java.sql.Timestamp; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.TimeZone; +import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; @@ -75,7 +75,6 @@ public class TestPrimitiveObjectInspectorUtils extends TestCase { @Test public void testgetTimestampWithMillisecondsInt() { - DateFormat localDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); DateFormat gmtDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); gmtDateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); @@ -85,69 +84,68 @@ public class TestPrimitiveObjectInspectorUtils extends TestCase { PrimitiveObjectInspector booleanOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.BOOLEAN); - assertEquals("1970-01-01 00:00:00.001", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(true, booleanOI))); - assertEquals("1970-01-01 00:00:00.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(false, booleanOI))); + assertEquals("1970-01-01 00:00:00.001", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(true, booleanOI).toSqlTimestamp())); + assertEquals("1970-01-01 00:00:00.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(false, booleanOI).toSqlTimestamp())); PrimitiveObjectInspector byteOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.BYTE); - assertEquals("1970-01-01 00:00:00.001", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((byte)1, byteOI))); - assertEquals("1969-12-31 23:59:59.999", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((byte)-1, byteOI))); + assertEquals("1970-01-01 00:00:00.001", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((byte)1, byteOI).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:59.999", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((byte)-1, byteOI).toSqlTimestamp())); PrimitiveObjectInspector shortOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.SHORT); - assertEquals("1970-01-01 00:00:00.001", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((short)1, shortOI))); - assertEquals("1969-12-31 23:59:59.999", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((short)-1, shortOI))); + assertEquals("1970-01-01 00:00:00.001", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((short)1, shortOI).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:59.999", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((short)-1, shortOI).toSqlTimestamp())); PrimitiveObjectInspector intOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.INT); - assertEquals("1970-01-17 11:22:01.282", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((int)1423321282, intOI))); - assertEquals("1969-12-31 23:59:59.999", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((int)-1, intOI))); + assertEquals("1970-01-17 11:22:01.282", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((int)1423321282, intOI).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:59.999", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((int)-1, intOI).toSqlTimestamp())); PrimitiveObjectInspector longOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.LONG); - assertEquals("1970-01-17 11:22:01.282", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(1423321282L, longOI))); - assertEquals("1969-12-31 23:59:59.999", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(-1L, longOI))); + assertEquals("1970-01-17 11:22:01.282", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(1423321282L, longOI).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:59.999", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(-1L, longOI).toSqlTimestamp())); // Float loses some precisions PrimitiveObjectInspector floatOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.FLOAT); - assertEquals("2015-02-07 15:02:24.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(1423321282.123f, floatOI))); - assertEquals("1969-12-31 23:59:58.876", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(-1.123f, floatOI))); + assertEquals("2015-02-07 15:02:24.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(1423321282.123f, floatOI).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:58.876", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(-1.123f, floatOI).toSqlTimestamp())); PrimitiveObjectInspector doubleOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.DOUBLE); - assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((double)1423321282.123, doubleOI))); - assertEquals("1969-12-31 23:59:58.877", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((double)-1.123, doubleOI))); + assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((double)1423321282.123, doubleOI).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:58.877", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((double)-1.123, doubleOI).toSqlTimestamp())); PrimitiveObjectInspector decimalOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.DECIMAL); - assertEquals("2015-02-07 15:01:22.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(HiveDecimal.create(1423321282L), decimalOI))); - assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(HiveDecimal.create(-1), decimalOI))); + assertEquals("2015-02-07 15:01:22.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(HiveDecimal.create(1423321282L), decimalOI).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(HiveDecimal.create(-1), decimalOI).toSqlTimestamp())); PrimitiveObjectInspector stringOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.STRING); - assertEquals("2015-02-07 15:01:22.123", localDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp("2015-02-07 15:01:22.123", stringOI))); + assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp("2015-02-07 15:01:22.123", stringOI).toSqlTimestamp())); PrimitiveObjectInspector charOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.CHAR); - assertEquals("2015-02-07 15:01:22.123", localDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new HiveChar("2015-02-07 15:01:22.123", 30), charOI))); + assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new HiveChar("2015-02-07 15:01:22.123", 30), charOI).toSqlTimestamp())); PrimitiveObjectInspector varcharOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.VARCHAR); - assertEquals("2015-02-07 15:01:22.123", localDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new HiveVarchar("2015-02-07 15:01:22.123",30), varcharOI))); + assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new HiveVarchar("2015-02-07 15:01:22.123",30), varcharOI).toSqlTimestamp())); PrimitiveObjectInspector dateOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.DATE); - assertEquals("2015-02-07 00:00:00.000", localDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new Date(1423321282123L), dateOI))); + assertEquals("2015-02-07 00:00:00.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(Date.ofEpochMilli(1423321282123L), dateOI).toSqlTimestamp())); PrimitiveObjectInspector timestampOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.TIMESTAMP); - assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new Timestamp(1423321282123L), timestampOI))); + assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(Timestamp.ofEpochMilli(1423321282123L), timestampOI).toSqlTimestamp())); } @Test public void testgetTimestampWithSecondsInt() { - DateFormat localDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); DateFormat gmtDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); gmtDateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); @@ -157,70 +155,72 @@ public class TestPrimitiveObjectInspectorUtils extends TestCase { PrimitiveObjectInspector booleanOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.BOOLEAN); - assertEquals("1970-01-01 00:00:01.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(true, booleanOI, true))); - assertEquals("1970-01-01 00:00:00.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(false, booleanOI, true))); + assertEquals("1970-01-01 00:00:01.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(true, booleanOI, true).toSqlTimestamp())); + assertEquals("1970-01-01 00:00:00.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(false, booleanOI, true).toSqlTimestamp())); PrimitiveObjectInspector byteOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.BYTE); - assertEquals("1970-01-01 00:00:01.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((byte)1, byteOI, true))); - assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((byte)-1, byteOI, true))); + assertEquals("1970-01-01 00:00:01.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((byte)1, byteOI, true).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((byte)-1, byteOI, true).toSqlTimestamp())); PrimitiveObjectInspector shortOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.SHORT); - assertEquals("1970-01-01 00:00:01.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((short)1, shortOI, true))); - assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((short)-1, shortOI, true))); + assertEquals("1970-01-01 00:00:01.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((short)1, shortOI, true).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((short)-1, shortOI, true).toSqlTimestamp())); PrimitiveObjectInspector intOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.INT); - assertEquals("2015-02-07 15:01:22.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((int)1423321282, intOI, true))); - assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((int)-1, intOI, true))); + assertEquals("2015-02-07 15:01:22.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((int)1423321282, intOI, true).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((int)-1, intOI, true).toSqlTimestamp())); PrimitiveObjectInspector longOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.LONG); - assertEquals("2015-02-07 15:01:22.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(1423321282L, longOI, true))); - assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(-1L, longOI, true))); + assertEquals("2015-02-07 15:01:22.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(1423321282L, longOI, true).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(-1L, longOI, true).toSqlTimestamp())); // Float loses some precisions PrimitiveObjectInspector floatOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.FLOAT); - assertEquals("2015-02-07 15:02:24.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(1423321282.123f, floatOI, true))); - assertEquals("1969-12-31 23:59:58.876", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(-1.123f, floatOI, true))); + assertEquals("2015-02-07 15:02:24.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(1423321282.123f, floatOI, true).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:58.876", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(-1.123f, floatOI, true).toSqlTimestamp())); PrimitiveObjectInspector doubleOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.DOUBLE); - assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((double)1423321282.123, doubleOI, true))); - assertEquals("1969-12-31 23:59:58.877", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((double)-1.123, doubleOI, true))); + assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((double)1423321282.123, doubleOI, true).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:58.877", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp((double)-1.123, doubleOI, true).toSqlTimestamp())); PrimitiveObjectInspector decimalOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.DECIMAL); - assertEquals("2015-02-07 15:01:22.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(HiveDecimal.create(1423321282L), decimalOI, true))); - assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(HiveDecimal.create(-1), decimalOI, true))); + assertEquals("2015-02-07 15:01:22.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(HiveDecimal.create(1423321282L), decimalOI, true).toSqlTimestamp())); + assertEquals("1969-12-31 23:59:59.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(HiveDecimal.create(-1), decimalOI, true).toSqlTimestamp())); PrimitiveObjectInspector stringOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.STRING); - assertEquals("2015-02-07 15:01:22.123", localDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp("2015-02-07 15:01:22.123", stringOI, true))); + assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp("2015-02-07 15:01:22.123", stringOI, true).toSqlTimestamp())); PrimitiveObjectInspector charOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.CHAR); - assertEquals("2015-02-07 15:01:22.123", localDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new HiveChar("2015-02-07 15:01:22.123", 30), charOI, true))); + assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new HiveChar("2015-02-07 15:01:22.123", 30), charOI, true).toSqlTimestamp())); PrimitiveObjectInspector varcharOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.VARCHAR); - assertEquals("2015-02-07 15:01:22.123", localDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new HiveVarchar("2015-02-07 15:01:22.123",30), varcharOI, true))); + assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new HiveVarchar("2015-02-07 15:01:22.123",30), varcharOI, true).toSqlTimestamp())); PrimitiveObjectInspector dateOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.DATE); - assertEquals("2015-02-07 00:00:00.000", localDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new Date(1423321282123L), dateOI, true))); + assertEquals("2015-02-07 00:00:00.000", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(Date.ofEpochMilli(1423321282123L), dateOI, true).toSqlTimestamp())); PrimitiveObjectInspector timestampOI = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveCategory.TIMESTAMP); - assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(new Timestamp(1423321282123L), timestampOI, true))); + assertEquals("2015-02-07 15:01:22.123", gmtDateFormat.format(PrimitiveObjectInspectorUtils.getTimestamp(Timestamp.ofEpochMilli(1423321282123L), timestampOI, true).toSqlTimestamp())); } @Test public void testGetTimestampFromString() { - DateFormat localDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); - assertEquals("2015-02-07 00:00:00.000", localDateFormat.format(PrimitiveObjectInspectorUtils.getTimestampFromString("2015-02-07"))); + DateFormat udfDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + udfDateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + assertEquals("2015-02-07 00:00:00.000", udfDateFormat.format( + PrimitiveObjectInspectorUtils.getTimestampFromString("2015-02-07").toSqlTimestamp())); } @Test
http://git-wip-us.apache.org/repos/asf/hive/blob/ae008b79/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 51e081b..f45b71f 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -1284,13 +1284,18 @@ class MetaStoreDirectSql { if (colType == FilterType.Date && valType == FilterType.String) { // Filter.g cannot parse a quoted date; try to parse date here too. try { - nodeValue = new java.sql.Date( - org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.PARTITION_DATE_FORMAT.get().parse((String)nodeValue).getTime()); + nodeValue = MetaStoreUtils.PARTITION_DATE_FORMAT.get().parse((String)nodeValue); valType = FilterType.Date; } catch (ParseException pe) { // do nothing, handled below - types will mismatch } } + // We format it so we are sure we are getting the right value + if (valType == FilterType.Date) { + // Format + nodeValue = MetaStoreUtils.PARTITION_DATE_FORMAT.get().format(nodeValue); + } + if (colType != valType) { // It's not clear how filtering for e.g. "stringCol > 5" should work (which side is // to be coerced?). Let the expression evaluation sort this one out, not metastore. http://git-wip-us.apache.org/repos/asf/hive/blob/ae008b79/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/Filter.g ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/Filter.g b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/Filter.g index 81111a0..4559383 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/Filter.g +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/Filter.g @@ -38,6 +38,7 @@ package org.apache.hadoop.hive.metastore.parser; import java.sql.Date; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.TimeZone; import java.util.regex.Matcher; import java.util.regex.Pattern; } @@ -52,6 +53,7 @@ import java.util.regex.Pattern; protected SimpleDateFormat initialValue() { SimpleDateFormat val = new SimpleDateFormat("yyyy-MM-dd"); val.setLenient(false); // Without this, 2020-20-20 becomes 2021-08-20. + val.setTimeZone(TimeZone.getTimeZone("UTC")); return val; }; }; http://git-wip-us.apache.org/repos/asf/hive/blob/ae008b79/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 5b58a08..cbe89b6 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -96,6 +96,7 @@ import java.util.Map.Entry; import java.util.SortedMap; import java.util.SortedSet; import java.util.StringJoiner; +import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.Callable; @@ -114,6 +115,7 @@ public class MetaStoreUtils { protected DateFormat initialValue() { DateFormat val = new SimpleDateFormat("yyyy-MM-dd"); val.setLenient(false); // Without this, 2020-20-20 becomes 2021-08-20. + val.setTimeZone(TimeZone.getTimeZone("UTC")); return val; } }; http://git-wip-us.apache.org/repos/asf/hive/blob/ae008b79/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java index 3b5f3ba..f11a319 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java @@ -81,7 +81,7 @@ public class TimestampColumnVector extends ColumnVector { scratchWritable = null; // Allocated by caller. - isUTC = false; + isUTC = true; } /** http://git-wip-us.apache.org/repos/asf/hive/blob/ae008b79/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java index 6325d5d..3894e09 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java +++ b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java @@ -38,6 +38,7 @@ import org.apache.hadoop.io.WritableUtils; * YYYY-MM-DD * */ +@Deprecated public class DateWritable implements WritableComparable<DateWritable> { private static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1); http://git-wip-us.apache.org/repos/asf/hive/blob/ae008b79/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java ---------------------------------------------------------------------- diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java index 3d9f262..fbd0c06 100644 --- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java +++ b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java @@ -110,7 +110,7 @@ public class TestStructColumnVector { batch.cols[0] = x; batch.cols[1] = y; batch.reset(); - Timestamp ts = Timestamp.valueOf("2000-01-01 00:00:00"); + Timestamp ts = new Timestamp(946684800000L); for(int r=0; r < 10; ++r) { batch.size += 1; x1.vector[r] = 3 * r; @@ -120,16 +120,16 @@ public class TestStructColumnVector { y.setRef(r, buffer, 0, buffer.length); } final String EXPECTED = ("Column vector types: 0:STRUCT<LONG, TIMESTAMP>, 1:BYTES\n" + - "[[0, 2000-01-01 00:00:01.0], \"value 0\"]\n" + - "[[3, 2000-01-01 00:00:02.0], \"value 1\"]\n" + - "[[6, 2000-01-01 00:00:03.0], \"value 2\"]\n" + - "[[9, 2000-01-01 00:00:04.0], \"value 3\"]\n" + - "[[12, 2000-01-01 00:00:05.0], \"value 4\"]\n" + - "[[15, 2000-01-01 00:00:06.0], \"value 5\"]\n" + - "[[18, 2000-01-01 00:00:07.0], \"value 6\"]\n" + - "[[21, 2000-01-01 00:00:08.0], \"value 7\"]\n" + - "[[24, 2000-01-01 00:00:09.0], \"value 8\"]\n" + - "[[27, 2000-01-01 00:00:10.0], \"value 9\"]"); + "[[0, 2000-01-01 00:00:01], \"value 0\"]\n" + + "[[3, 2000-01-01 00:00:02], \"value 1\"]\n" + + "[[6, 2000-01-01 00:00:03], \"value 2\"]\n" + + "[[9, 2000-01-01 00:00:04], \"value 3\"]\n" + + "[[12, 2000-01-01 00:00:05], \"value 4\"]\n" + + "[[15, 2000-01-01 00:00:06], \"value 5\"]\n" + + "[[18, 2000-01-01 00:00:07], \"value 6\"]\n" + + "[[21, 2000-01-01 00:00:08], \"value 7\"]\n" + + "[[24, 2000-01-01 00:00:09], \"value 8\"]\n" + + "[[27, 2000-01-01 00:00:10], \"value 9\"]"); assertEquals(EXPECTED, batch.toString()); } http://git-wip-us.apache.org/repos/asf/hive/blob/ae008b79/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java ---------------------------------------------------------------------- diff --git a/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java b/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java index 6d7ed3e..8b36371 100644 --- a/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java +++ b/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java @@ -1589,14 +1589,14 @@ public class GenVectorCode extends Task { vectorType = "long"; getPrimitiveMethod = "getDate"; getValueMethod = ""; - conversionMethod = "DateWritable.dateToDays"; + conversionMethod = "DateWritableV2.dateToDays"; // Special case - Date requires its own specific BetweenDynamicValue class, but derives from FilterLongColumnBetween typeName = "Long"; } else if (operandType.equals("timestamp")) { defaultValue = "new Timestamp(0)"; vectorType = "Timestamp"; getPrimitiveMethod = "getTimestamp"; - getValueMethod = ""; + getValueMethod = ".toSqlTimestamp()"; conversionMethod = ""; } else { throw new IllegalArgumentException("Type " + operandType + " not supported"); @@ -3159,8 +3159,7 @@ public class GenVectorCode extends Task { private String getDTIScalarColumnDisplayBody(String type) { if (type.equals("date")) { return - "Date dt = new Date(0);" + - " dt.setTime(DateWritable.daysToMillis((int) value));\n" + + "Date dt = Date.ofEpochMilli(DateWritableV2.daysToMillis((int) value));\n" + " return \"date \" + dt.toString() + \", \" + getColumnParamString(0, colNum);"; } else { return @@ -3171,8 +3170,7 @@ public class GenVectorCode extends Task { private String getDTIColumnScalarDisplayBody(String type) { if (type.equals("date")) { return - "Date dt = new Date(0);" + - " dt.setTime(DateWritable.daysToMillis((int) value));\n" + + "Date dt = Date.ofEpochMilli(DateWritableV2.daysToMillis((int) value));\n" + " return getColumnParamString(0, colNum) + \", date \" + dt.toString();"; } else { return @@ -3842,9 +3840,9 @@ public class GenVectorCode extends Task { } else if (primitiveType.equals("interval_day_time")) { return "HiveIntervalDayTimeWritable"; } else if (primitiveType.equals("date")) { - return "HiveDateWritable"; + return "DateWritableV2"; } else if (primitiveType.equals("timestamp")) { - return "HiveTimestampWritable"; + return "TimestampWritable"; } throw new Exception("Unimplemented primitive output writable: " + primitiveType); }
