[hive] branch branch-2.3 updated: HIVE-21200: Vectorization: date column throwing java.lang.UnsupportedOperationException for parquet (#2276)

[sunchao]

This is an automated email from the ASF dual-hosted git repository.

sunchao pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/branch-2.3 by this push:
     new 9880ee6  HIVE-21200: Vectorization: date column throwing 
java.lang.UnsupportedOperationException for parquet (#2276)
9880ee6 is described below

commit 9880ee69b327efbd791d5b021115fee3e51903d2
Author: Daoyuan Wang <m...@daoyuan.wang>
AuthorDate: Sun May 16 01:15:50 2021 +0800

    HIVE-21200: Vectorization: date column throwing 
java.lang.UnsupportedOperationException for parquet (#2276)
---
 .../vector/VectorizedPrimitiveColumnReader.java    |  2 +-
 .../ql/io/parquet/TestVectorizedColumnReader.java  |  5 ++++
 .../io/parquet/VectorizedColumnReaderTestBase.java | 33 +++++++++++++++++++++-
 3 files changed, 38 insertions(+), 2 deletions(-)

diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java
index 3d5c6e6..a0e9484 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java
@@ -168,9 +168,9 @@ public class VectorizedPrimitiveColumnReader implements 
VectorizedColumnReader {
     case INT:
     case BYTE:
     case SHORT:
+    case DATE:
       readIntegers(num, (LongColumnVector) column, rowId);
       break;
-    case DATE:
     case INTERVAL_YEAR_MONTH:
     case LONG:
       readLongs(num, (LongColumnVector) column, rowId);
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java
 
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java
index 670bfa6..435acda 100644
--- 
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java
+++ 
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReader.java
@@ -57,6 +57,11 @@ public class TestVectorizedColumnReader extends 
VectorizedColumnReaderTestBase {
   }
 
   @Test
+  public void testDateRead() throws Exception {
+    dateRead(isDictionaryEncoding);
+  }
+
+  @Test
   public void testDoubleRead() throws Exception {
     doubleRead(isDictionaryEncoding);
   }
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java
 
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java
index f537cee..8fce9f1 100644
--- 
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java
+++ 
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/VectorizedColumnReaderTestBase.java
@@ -79,6 +79,7 @@ public class VectorizedColumnReaderTestBase {
       + "required int32 int32_field; "
       + "required int64 int64_field; "
       + "required int96 int96_field; "
+      + "required int32 date_field; "
       + "required double double_field; "
       + "required float float_field; "
       + "required boolean boolean_field; "
@@ -246,7 +247,8 @@ public class VectorizedColumnReaderTestBase {
         .append("double_field", doubleVal)
         .append("float_field", floatVal)
         .append("boolean_field", booleanVal)
-        .append("flba_field", "abc");
+        .append("flba_field", "abc")
+        .append("date_field", intVal);
 
       if (!isNull) {
         group.append("some_null_field", "x");
@@ -372,6 +374,35 @@ public class VectorizedColumnReaderTestBase {
     }
   }
 
+  protected void dateRead(boolean isDictionaryEncoding) throws 
InterruptedException, HiveException, IOException {
+    Configuration conf = new Configuration();
+    conf.set(IOConstants.COLUMNS,"date_field");
+    conf.set(IOConstants.COLUMNS_TYPES,"date");
+    conf.setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false);
+    conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0");
+    VectorizedParquetRecordReader reader =
+            createParquetReader("message test { required int date_field;}", 
conf);
+    VectorizedRowBatch previous = reader.createValue();
+    try {
+      int c = 0;
+      while (reader.next(NullWritable.get(), previous)) {
+        LongColumnVector vector = (LongColumnVector) previous.cols[0];
+        assertTrue(vector.noNulls);
+        for (int i = 0; i < vector.vector.length; i++) {
+          if (c == nElements){
+            break;
+          }
+          assertEquals("Failed at " + c, getIntValue(isDictionaryEncoding, c), 
vector.vector[i]);
+          assertFalse(vector.isNull[i]);
+          c++;
+        }
+      }
+      assertEquals(nElements, c);
+    } finally {
+      reader.close();
+    }
+  }
+
   protected void doubleRead(boolean isDictionaryEncoding) throws Exception {
     Configuration conf = new Configuration();
     conf.set(IOConstants.COLUMNS, "double_field");