This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 70562437d36 HIVE-26320: Incorrect results for IN UDF on Parquet column
of CHAR/VARCHAR type (John Sherman reviewed by Aman Sinha, Krisztian Kasa,
Stamatis Zampetakis, Alessandro Solimando, Dayakar)
70562437d36 is described below
commit 70562437d369c2f4ab3e879bae519f81d386da3b
Author: John Sherman <j...@cloudera.com>
AuthorDate: Tue Sep 27 09:42:51 2022 -0700
HIVE-26320: Incorrect results for IN UDF on Parquet column of CHAR/VARCHAR
type (John Sherman reviewed by Aman Sinha, Krisztian Kasa, Stamatis Zampetakis,
Alessandro Solimando, Dayakar)
Closes #3628
---
.../hive/ql/io/parquet/convert/ETypeConverter.java | 32 +++-
.../ql/io/parquet/convert/TestETypeConverter.java | 65 +++++--
ql/src/test/queries/clientpositive/pointlookup.q | 33 ++++
ql/src/test/queries/clientpositive/udf_in.q | 6 +
.../results/clientpositive/llap/pointlookup.q.out | 193 +++++++++++++++++++++
.../test/results/clientpositive/llap/udf_in.q.out | 37 ++++
.../hadoop/hive/serde2/io/HiveCharWritable.java | 9 +
.../hadoop/hive/serde2/io/HiveVarcharWritable.java | 9 +
8 files changed, 372 insertions(+), 12 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
index 40069cf8b0c..28207714e3c 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
@@ -36,12 +36,17 @@ import
org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriteSupport;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.io.DateWritableV2;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.FloatWritable;
@@ -481,7 +486,32 @@ public enum ETypeConverter {
},
ESTRING_CONVERTER(String.class) {
@Override
- PrimitiveConverter getConverter(final PrimitiveType type, final int index,
final ConverterParent parent, TypeInfo hiveTypeInfo) {
+ PrimitiveConverter getConverter(final PrimitiveType type, final int index,
final ConverterParent parent,
+ TypeInfo hiveTypeInfo) {
+ // If we have type information, we should return properly typed strings.
However, there are a variety
+ // of code paths that do not provide the typeInfo in those cases we
default to Text. This idiom is also
+ // followed by for example the BigDecimal converter in which if there is
no type information,
+ // it defaults to the widest representation
+ if (hiveTypeInfo instanceof PrimitiveTypeInfo) {
+ PrimitiveTypeInfo t = (PrimitiveTypeInfo) hiveTypeInfo;
+ switch (t.getPrimitiveCategory()) {
+ case CHAR:
+ return new BinaryConverter<HiveCharWritable>(type, parent, index) {
+ @Override
+ protected HiveCharWritable convert(Binary binary) {
+ return new HiveCharWritable(binary.getBytes(), ((CharTypeInfo)
hiveTypeInfo).getLength());
+ }
+ };
+ case VARCHAR:
+ return new BinaryConverter<HiveVarcharWritable>(type, parent,
index) {
+ @Override
+ protected HiveVarcharWritable convert(Binary binary) {
+ return new HiveVarcharWritable(binary.getBytes(),
((VarcharTypeInfo) hiveTypeInfo).getLength());
+ }
+ };
+ }
+ }
+ // STRING type
return new BinaryConverter<Text>(type, parent, index) {
@Override
protected Text convert(Binary binary) {
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java
index 8161430a501..fcfb5c7782c 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.io.parquet.convert;
+import static
org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.stringTypeInfo;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -30,8 +31,11 @@ import org.apache.hadoop.hive.common.type.Timestamp;
import
org.apache.hadoop.hive.ql.io.parquet.convert.ETypeConverter.BinaryConverter;
import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime;
import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -212,23 +216,62 @@ public class TestETypeConverter {
}
@Test
- public void testGetTextConverter() throws Exception {
+ public void testGetTextConverterForString() throws Exception {
PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.BINARY)
- .as(LogicalTypeAnnotation.stringType()).named("value");
- Writable writable = getWritableFromBinaryConverter(new VarcharTypeInfo(),
primitiveType,
- Binary.fromString("this_is_a_value"));
- Text textWritable = (Text) writable;
- assertEquals("this_is_a_value", textWritable.toString());
+ .as(LogicalTypeAnnotation.stringType()).named("value");
+ String value = "this_is_a_value";
+ Text textWritable = (Text) getWritableFromBinaryConverter(stringTypeInfo,
primitiveType,
+ Binary.fromString(value));
+ assertEquals(value, textWritable.toString());
+ }
+
+ @Test
+ public void testGetTextConverterForCharPadsValueWithSpacesTillLen() {
+ PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.stringType()).named("value");
+ String value = "this_is_a_value";
+ HiveCharWritable textWritable = (HiveCharWritable)
getWritableFromBinaryConverter(
+ new CharTypeInfo(value.length() + 2), primitiveType,
Binary.fromString(value));
+ assertEquals(value + " ", textWritable.toString());
+ }
+
+ @Test
+ public void testGetTextConverterForCharTruncatesValueExceedingLen() {
+ PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.stringType()).named("value");
+ String value = "this_is_a_value";
+ HiveCharWritable textWritable = (HiveCharWritable)
getWritableFromBinaryConverter(
+ new CharTypeInfo(6), primitiveType, Binary.fromString(value));
+ assertEquals(value.substring(0, 6), textWritable.toString());
}
@Test
- public void testGetTextConverterNoHiveTypeInfo() throws Exception {
+ public void testGetTextConverterForVarcharTruncatesValueExceedingLen() {
+ PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.stringType()).named("value");
+ String value = "this_is_a_value";
+ HiveVarcharWritable textWritable = (HiveVarcharWritable)
getWritableFromBinaryConverter(
+ new VarcharTypeInfo(6), primitiveType, Binary.fromString(value));
+ assertEquals(value.substring(0, 6), textWritable.toString());
+ }
+
+ @Test
+ public void testGetTextConverterForVarchar() {
+ PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.stringType()).named("value");
+ String value = "this_is_a_value";
+ HiveVarcharWritable textWritable = (HiveVarcharWritable)
getWritableFromBinaryConverter(
+ new VarcharTypeInfo(34), primitiveType, Binary.fromString(value));
+ assertEquals(value, textWritable.toString());
+ }
+
+ @Test
+ public void testGetTextConverterNoHiveTypeInfo() {
PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.BINARY)
.as(LogicalTypeAnnotation.stringType()).named("value");
- Writable writable =
- getWritableFromBinaryConverter(null, primitiveType,
Binary.fromString("this_is_a_value"));
- Text textWritable = (Text) writable;
- assertEquals("this_is_a_value", textWritable.toString());
+ String value = "this_is_a_value";
+ Text textWritable = (Text) getWritableFromBinaryConverter(null,
primitiveType, Binary.fromString(value));
+ assertEquals(value, textWritable.toString());
}
@Test
diff --git a/ql/src/test/queries/clientpositive/pointlookup.q
b/ql/src/test/queries/clientpositive/pointlookup.q
index a808a51cbfc..0853db0b69e 100644
--- a/ql/src/test/queries/clientpositive/pointlookup.q
+++ b/ql/src/test/queries/clientpositive/pointlookup.q
@@ -127,3 +127,36 @@ or inOutputOpt.key = null;
drop table orOutput;
drop table inOutput;
drop table inOutputOpt;
+
+-- test case(s) for HIVE-26320 for ORC
+SET hive.optimize.point.lookup=true;
+SET hive.optimize.point.lookup.min=2;
+CREATE EXTERNAL TABLE hive26230_orc(kob varchar(2), enhanced_type_code int)
STORED AS ORC;
+INSERT INTO hive26230_orc VALUES('BB',18),('BC',18),('AB',18);
+SELECT CASE WHEN ((kob='BB' AND enhanced_type_code=18) OR (kob='BC' AND
enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_orc;
+DROP TABLE hive26230_orc;
+
+CREATE EXTERNAL TABLE hive26230_char_orc(kob char(2), enhanced_type_code int)
STORED AS ORC;
+INSERT INTO hive26230_char_orc VALUES('B',18),('BC',18),('AB',18);
+SELECT CASE WHEN ((kob='B' AND enhanced_type_code=18) OR (kob='BC' AND
enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_char_orc;
+DROP TABLE hive26230_char_orc;
+
+-- test case(s) for HIVE-26320 for PARQUET
+CREATE EXTERNAL TABLE hive26230_parq(kob varchar(2), enhanced_type_code int)
STORED AS PARQUET;
+INSERT INTO hive26230_parq VALUES('BB',18),('BC',18),('AB',18);
+SELECT CASE WHEN ((kob='BB' AND enhanced_type_code=18) OR (kob='BC' AND
enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_parq;
+DROP TABLE hive26230_parq;
+
+CREATE EXTERNAL TABLE hive26230_char_parq(kob char(2), enhanced_type_code int)
STORED AS PARQUET;
+INSERT INTO hive26230_char_parq VALUES('B',18),('BC',18),('AB',18);
+SELECT CASE WHEN ((kob='B' AND enhanced_type_code=18) OR (kob='BC' AND
enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_char_parq;
+DROP TABLE hive26230_char_parq;
+
+CREATE EXTERNAL TABLE hive26230_int_parq(kob int, enhanced_type_code int)
STORED AS PARQUET;
+INSERT INTO hive26230_int_parq VALUES(2,18),(23,18),(12,18);
+SELECT CASE WHEN ((kob=2 AND enhanced_type_code=18) OR (kob=23 AND
enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check FROM
hive26230_int_parq;
+DROP TABLE hive26230_int_parq;
diff --git a/ql/src/test/queries/clientpositive/udf_in.q
b/ql/src/test/queries/clientpositive/udf_in.q
index 99fafd44bfa..7a6e98f2ca5 100644
--- a/ql/src/test/queries/clientpositive/udf_in.q
+++ b/ql/src/test/queries/clientpositive/udf_in.q
@@ -14,3 +14,9 @@ SELECT 1 IN (1, 2, 3),
(1+3) IN (5, 6, (1+2) + 1) FROM src tablesample (1 rows);
SELECT key FROM src WHERE key IN ("238", 86);
+
+-- test case(s) for HIVE-26320
+CREATE EXTERNAL TABLE hive26230_parq_in(kob varchar(2), code int) STORED AS
PARQUET;
+INSERT INTO hive26230_parq_in VALUES('BB',18),('BC',18),('AB',18);
+SELECT (kob, code) IN ((cast('BB' as varchar(2)), 18), (cast('BC' as
varchar(2)), 18)) FROM hive26230_parq_in;
+DROP TABLE hive26230_parq_in;
diff --git a/ql/src/test/results/clientpositive/llap/pointlookup.q.out
b/ql/src/test/results/clientpositive/llap/pointlookup.q.out
index d4aef2f7a81..25de4bb82ba 100644
--- a/ql/src/test/results/clientpositive/llap/pointlookup.q.out
+++ b/ql/src/test/results/clientpositive/llap/pointlookup.q.out
@@ -362,3 +362,196 @@ POSTHOOK: query: drop table inOutputOpt
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@inoutputopt
POSTHOOK: Output: default@inoutputopt
+PREHOOK: query: CREATE EXTERNAL TABLE hive26230_orc(kob varchar(2),
enhanced_type_code int) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hive26230_orc
+POSTHOOK: query: CREATE EXTERNAL TABLE hive26230_orc(kob varchar(2),
enhanced_type_code int) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hive26230_orc
+PREHOOK: query: INSERT INTO hive26230_orc VALUES('BB',18),('BC',18),('AB',18)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hive26230_orc
+POSTHOOK: query: INSERT INTO hive26230_orc VALUES('BB',18),('BC',18),('AB',18)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hive26230_orc
+POSTHOOK: Lineage: hive26230_orc.enhanced_type_code SCRIPT []
+POSTHOOK: Lineage: hive26230_orc.kob SCRIPT []
+PREHOOK: query: SELECT CASE WHEN ((kob='BB' AND enhanced_type_code=18) OR
(kob='BC' AND enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_orc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hive26230_orc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT CASE WHEN ((kob='BB' AND enhanced_type_code=18) OR
(kob='BC' AND enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_orc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hive26230_orc
+#### A masked pattern was here ####
+1
+1
+0
+PREHOOK: query: DROP TABLE hive26230_orc
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hive26230_orc
+PREHOOK: Output: default@hive26230_orc
+POSTHOOK: query: DROP TABLE hive26230_orc
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hive26230_orc
+POSTHOOK: Output: default@hive26230_orc
+PREHOOK: query: CREATE EXTERNAL TABLE hive26230_char_orc(kob char(2),
enhanced_type_code int) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hive26230_char_orc
+POSTHOOK: query: CREATE EXTERNAL TABLE hive26230_char_orc(kob char(2),
enhanced_type_code int) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hive26230_char_orc
+PREHOOK: query: INSERT INTO hive26230_char_orc
VALUES('B',18),('BC',18),('AB',18)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hive26230_char_orc
+POSTHOOK: query: INSERT INTO hive26230_char_orc
VALUES('B',18),('BC',18),('AB',18)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hive26230_char_orc
+POSTHOOK: Lineage: hive26230_char_orc.enhanced_type_code SCRIPT []
+POSTHOOK: Lineage: hive26230_char_orc.kob SCRIPT []
+PREHOOK: query: SELECT CASE WHEN ((kob='B' AND enhanced_type_code=18) OR
(kob='BC' AND enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_char_orc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hive26230_char_orc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT CASE WHEN ((kob='B' AND enhanced_type_code=18) OR
(kob='BC' AND enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_char_orc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hive26230_char_orc
+#### A masked pattern was here ####
+1
+1
+0
+PREHOOK: query: DROP TABLE hive26230_char_orc
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hive26230_char_orc
+PREHOOK: Output: default@hive26230_char_orc
+POSTHOOK: query: DROP TABLE hive26230_char_orc
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hive26230_char_orc
+POSTHOOK: Output: default@hive26230_char_orc
+PREHOOK: query: CREATE EXTERNAL TABLE hive26230_parq(kob varchar(2),
enhanced_type_code int) STORED AS PARQUET
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hive26230_parq
+POSTHOOK: query: CREATE EXTERNAL TABLE hive26230_parq(kob varchar(2),
enhanced_type_code int) STORED AS PARQUET
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hive26230_parq
+PREHOOK: query: INSERT INTO hive26230_parq VALUES('BB',18),('BC',18),('AB',18)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hive26230_parq
+POSTHOOK: query: INSERT INTO hive26230_parq VALUES('BB',18),('BC',18),('AB',18)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hive26230_parq
+POSTHOOK: Lineage: hive26230_parq.enhanced_type_code SCRIPT []
+POSTHOOK: Lineage: hive26230_parq.kob SCRIPT []
+PREHOOK: query: SELECT CASE WHEN ((kob='BB' AND enhanced_type_code=18) OR
(kob='BC' AND enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_parq
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hive26230_parq
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT CASE WHEN ((kob='BB' AND enhanced_type_code=18) OR
(kob='BC' AND enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_parq
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hive26230_parq
+#### A masked pattern was here ####
+1
+1
+0
+PREHOOK: query: DROP TABLE hive26230_parq
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hive26230_parq
+PREHOOK: Output: default@hive26230_parq
+POSTHOOK: query: DROP TABLE hive26230_parq
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hive26230_parq
+POSTHOOK: Output: default@hive26230_parq
+PREHOOK: query: CREATE EXTERNAL TABLE hive26230_char_parq(kob char(2),
enhanced_type_code int) STORED AS PARQUET
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hive26230_char_parq
+POSTHOOK: query: CREATE EXTERNAL TABLE hive26230_char_parq(kob char(2),
enhanced_type_code int) STORED AS PARQUET
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hive26230_char_parq
+PREHOOK: query: INSERT INTO hive26230_char_parq
VALUES('B',18),('BC',18),('AB',18)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hive26230_char_parq
+POSTHOOK: query: INSERT INTO hive26230_char_parq
VALUES('B',18),('BC',18),('AB',18)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hive26230_char_parq
+POSTHOOK: Lineage: hive26230_char_parq.enhanced_type_code SCRIPT []
+POSTHOOK: Lineage: hive26230_char_parq.kob SCRIPT []
+PREHOOK: query: SELECT CASE WHEN ((kob='B' AND enhanced_type_code=18) OR
(kob='BC' AND enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_char_parq
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hive26230_char_parq
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT CASE WHEN ((kob='B' AND enhanced_type_code=18) OR
(kob='BC' AND enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check
+FROM hive26230_char_parq
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hive26230_char_parq
+#### A masked pattern was here ####
+1
+1
+0
+PREHOOK: query: DROP TABLE hive26230_char_parq
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hive26230_char_parq
+PREHOOK: Output: default@hive26230_char_parq
+POSTHOOK: query: DROP TABLE hive26230_char_parq
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hive26230_char_parq
+POSTHOOK: Output: default@hive26230_char_parq
+PREHOOK: query: CREATE EXTERNAL TABLE hive26230_int_parq(kob int,
enhanced_type_code int) STORED AS PARQUET
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hive26230_int_parq
+POSTHOOK: query: CREATE EXTERNAL TABLE hive26230_int_parq(kob int,
enhanced_type_code int) STORED AS PARQUET
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hive26230_int_parq
+PREHOOK: query: INSERT INTO hive26230_int_parq VALUES(2,18),(23,18),(12,18)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hive26230_int_parq
+POSTHOOK: query: INSERT INTO hive26230_int_parq VALUES(2,18),(23,18),(12,18)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hive26230_int_parq
+POSTHOOK: Lineage: hive26230_int_parq.enhanced_type_code SCRIPT []
+POSTHOOK: Lineage: hive26230_int_parq.kob SCRIPT []
+PREHOOK: query: SELECT CASE WHEN ((kob=2 AND enhanced_type_code=18) OR (kob=23
AND enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check FROM
hive26230_int_parq
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hive26230_int_parq
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT CASE WHEN ((kob=2 AND enhanced_type_code=18) OR
(kob=23 AND enhanced_type_code=18)) THEN 1 ELSE 0 END AS logic_check FROM
hive26230_int_parq
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hive26230_int_parq
+#### A masked pattern was here ####
+1
+1
+0
+PREHOOK: query: DROP TABLE hive26230_int_parq
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hive26230_int_parq
+PREHOOK: Output: default@hive26230_int_parq
+POSTHOOK: query: DROP TABLE hive26230_int_parq
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hive26230_int_parq
+POSTHOOK: Output: default@hive26230_int_parq
diff --git a/ql/src/test/results/clientpositive/llap/udf_in.q.out
b/ql/src/test/results/clientpositive/llap/udf_in.q.out
index 85ee95108a8..8c840eda218 100644
--- a/ql/src/test/results/clientpositive/llap/udf_in.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_in.q.out
@@ -38,3 +38,40 @@ POSTHOOK: Input: default@src
238
86
238
+PREHOOK: query: CREATE EXTERNAL TABLE hive26230_parq_in(kob varchar(2), code
int) STORED AS PARQUET
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hive26230_parq_in
+POSTHOOK: query: CREATE EXTERNAL TABLE hive26230_parq_in(kob varchar(2), code
int) STORED AS PARQUET
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hive26230_parq_in
+PREHOOK: query: INSERT INTO hive26230_parq_in
VALUES('BB',18),('BC',18),('AB',18)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@hive26230_parq_in
+POSTHOOK: query: INSERT INTO hive26230_parq_in
VALUES('BB',18),('BC',18),('AB',18)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@hive26230_parq_in
+POSTHOOK: Lineage: hive26230_parq_in.code SCRIPT []
+POSTHOOK: Lineage: hive26230_parq_in.kob SCRIPT []
+PREHOOK: query: SELECT (kob, code) IN ((cast('BB' as varchar(2)), 18),
(cast('BC' as varchar(2)), 18)) FROM hive26230_parq_in
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hive26230_parq_in
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT (kob, code) IN ((cast('BB' as varchar(2)), 18),
(cast('BC' as varchar(2)), 18)) FROM hive26230_parq_in
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hive26230_parq_in
+#### A masked pattern was here ####
+true
+true
+false
+PREHOOK: query: DROP TABLE hive26230_parq_in
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hive26230_parq_in
+PREHOOK: Output: default@hive26230_parq_in
+POSTHOOK: query: DROP TABLE hive26230_parq_in
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hive26230_parq_in
+POSTHOOK: Output: default@hive26230_parq_in
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java
b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java
index fcd14e5ec87..01448ebe8a5 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java
@@ -42,6 +42,15 @@ public class HiveCharWritable extends HiveBaseCharWritable
set(hcw);
}
+ public HiveCharWritable(byte[] bytes, int maxLength) {
+ set(bytes, maxLength);
+ }
+
+ public void set(byte[] bytes, int maxLength) {
+ value.set(bytes);
+ enforceMaxLength(maxLength);
+ }
+
public void set(HiveChar val) {
set(val.getValue(), -1);
}
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java
b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java
index c3812d62e11..8c247105ad8 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java
@@ -35,6 +35,15 @@ public class HiveVarcharWritable extends HiveBaseCharWritable
set(hcw);
}
+ public HiveVarcharWritable(byte[] bytes, int maxLength) {
+ set(bytes, maxLength);
+ }
+
+ public void set(byte[] bytes, int maxLength) {
+ value.set(bytes);
+ enforceMaxLength(maxLength);
+ }
+
public void set(HiveVarchar val) {
set(val.getValue());
}
