HIVE-11484 : Fix ObjectInspector for Char and VarChar for null and string objects (Deepak Barr, reviewed by Amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ddab69c4 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ddab69c4 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ddab69c4 Branch: refs/heads/llap Commit: ddab69c495ce6c325d7e06cfb0c704546e0d56b9 Parents: 94c1974 Author: Deepak Barr <deepak.b...@gmail.com> Authored: Wed Apr 13 09:50:24 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Wed Apr 13 09:50:24 2016 +0530 ---------------------------------------------------------------------- .../hadoop/hive/common/type/HiveBaseChar.java | 6 ++ .../hadoop/hive/common/type/HiveVarchar.java | 4 +- .../hive/common/type/TestHiveBaseChar.java | 2 + .../primitive/JavaHiveCharObjectInspector.java | 15 +++- .../JavaHiveVarcharObjectInspector.java | 15 +++- .../PrimitiveObjectInspectorConverter.java | 8 +- .../TestObjectInspectorConverters.java | 95 ++++++++++++++++++++ .../TestStandardObjectInspectors.java | 14 ++- 8 files changed, 148 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java index 3514726..53684e7 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java @@ -38,6 +38,9 @@ public abstract class HiveBaseChar { } public static String enforceMaxLength(String val, int maxLength) { + if (val == null) { + return null; + } String value = val; if (maxLength > 0) { @@ -52,6 +55,9 @@ public abstract class HiveBaseChar { } public static String getPaddedValue(String val, int maxLength) { + if (val == null) { + return null; + } if (maxLength < 0) { return val; } http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java index 969d474..09009eb 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java @@ -58,10 +58,10 @@ public class HiveVarchar extends HiveBaseChar return this.getValue().compareTo(rhs.getValue()); } - public boolean equals(HiveVarchar rhs) { + public boolean equals(Object rhs) { if (rhs == this) { return true; } - return this.getValue().equals(rhs.getValue()); + return this.getValue().equals(((HiveVarchar)rhs).getValue()); } } http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java index 2848465..98ad74a 100644 --- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java +++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java @@ -80,6 +80,7 @@ public class TestHiveBaseChar extends TestCase { assertEquals(strLen, enforcedString.codePointCount(0, enforcedString.length())); } } + assertNull(HiveBaseChar.enforceMaxLength(null, 0)); } public void testGetPaddedValue() { @@ -96,5 +97,6 @@ public class TestHiveBaseChar extends TestCase { assertEquals("abc ", HiveBaseChar.getPaddedValue("abc", 10)); assertEquals("abc ", HiveBaseChar.getPaddedValue("abc ", 10)); + assertNull(HiveBaseChar.getPaddedValue(null, 0)); } } http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java index f429709..3d89c92 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java @@ -37,7 +37,12 @@ public class JavaHiveCharObjectInspector extends AbstractPrimitiveJavaObjectInsp if (o == null) { return null; } - HiveChar value = (HiveChar) o; + HiveChar value; + if (o instanceof String) { + value = new HiveChar((String) o, getMaxLength()); + } else { + value = (HiveChar) o; + } if (BaseCharUtils.doesPrimitiveMatchTypeParams(value, (CharTypeInfo) typeInfo)) { return value; } @@ -49,7 +54,13 @@ public class JavaHiveCharObjectInspector extends AbstractPrimitiveJavaObjectInsp if (o == null) { return null; } - return getWritableWithParams((HiveChar) o); + HiveChar var; + if (o instanceof String) { + var = new HiveChar((String) o, getMaxLength()); + } else { + var = (HiveChar) o; + } + return getWritableWithParams(var); } private HiveChar getPrimitiveWithParams(HiveChar val) { http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java index a8e34ff..2320d2d 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java @@ -38,7 +38,12 @@ public class JavaHiveVarcharObjectInspector extends AbstractPrimitiveJavaObjectI if (o == null) { return null; } - HiveVarchar value = (HiveVarchar) o; + HiveVarchar value; + if (o instanceof String) { + value= new HiveVarchar((String)o, getMaxLength()); + } else { + value = (HiveVarchar)o; + } if (BaseCharUtils.doesPrimitiveMatchTypeParams(value, (VarcharTypeInfo) typeInfo)) { return value; } @@ -51,7 +56,13 @@ public class JavaHiveVarcharObjectInspector extends AbstractPrimitiveJavaObjectI if (o == null) { return null; } - return getWritableWithParams((HiveVarchar) o); + HiveVarchar var; + if (o instanceof String) { + var= new HiveVarchar((String)o, getMaxLength()); + } else { + var = (HiveVarchar)o; + } + return getWritableWithParams(var); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java index 5eb41d5..e08ad43 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java @@ -514,7 +514,7 @@ public class PrimitiveObjectInspectorConverter { PrimitiveObjectInspector inputOI; SettableHiveVarcharObjectInspector outputOI; - HiveVarcharWritable hc; + Object hc; public HiveVarcharConverter(PrimitiveObjectInspector inputOI, SettableHiveVarcharObjectInspector outputOI) { @@ -528,7 +528,7 @@ public class PrimitiveObjectInspectorConverter { //if (typeParams == null) { // throw new RuntimeException("varchar type used without type params"); //} - hc = new HiveVarcharWritable(); + hc = outputOI.create(new HiveVarchar("",-1)); } @Override @@ -551,13 +551,13 @@ public class PrimitiveObjectInspectorConverter { public static class HiveCharConverter implements Converter { PrimitiveObjectInspector inputOI; SettableHiveCharObjectInspector outputOI; - HiveCharWritable hc; + Object hc; public HiveCharConverter(PrimitiveObjectInspector inputOI, SettableHiveCharObjectInspector outputOI) { this.inputOI = inputOI; this.outputOI = outputOI; - hc = new HiveCharWritable(); + hc = outputOI.create(new HiveChar("",-1)); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java ---------------------------------------------------------------------- diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java index dd18517..2e1bb22 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java @@ -22,10 +22,15 @@ import java.util.List; import junit.framework.TestCase; +import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveCharWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -116,6 +121,96 @@ public class TestObjectInspectorConverters extends TestCase { .convert(Integer.valueOf(1))); assertEquals("DoubleConverter", null, doubleConverter.convert(null)); + // Char + Converter charConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, + PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector); + assertEquals("CharConverter", new HiveChar("TRUE", -1), charConverter + .convert(Boolean.valueOf(true))); + assertEquals("CharConverter", new HiveChar("FALSE", -1), charConverter + .convert(Boolean.valueOf(false))); + + charConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, + PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector); + assertEquals("CharConverter", new HiveCharWritable(new HiveChar("TRUE", -1)), charConverter + .convert(Boolean.valueOf(true))); + assertEquals("CharConverter", new HiveCharWritable(new HiveChar("FALSE", -1)), charConverter + .convert(Boolean.valueOf(false))); + + charConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaIntObjectInspector, + PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector); + assertEquals("CharConverter", new HiveChar("0", -1), charConverter + .convert(Integer.valueOf(0))); + assertEquals("CharConverter", new HiveChar("1", -1), charConverter + .convert(Integer.valueOf(1))); + + charConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaIntObjectInspector, + PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector); + assertEquals("CharConverter", new HiveCharWritable(new HiveChar("0", -1)), charConverter + .convert(Integer.valueOf(0))); + assertEquals("CharConverter", new HiveCharWritable(new HiveChar("1", -1)), charConverter + .convert(Integer.valueOf(1))); + + charConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector); + assertEquals("CharConverter", new HiveChar("hive", -1), charConverter + .convert(String.valueOf("hive"))); + + charConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector); + assertEquals("CharConverter", new HiveCharWritable(new HiveChar("hive", -1)), charConverter + .convert(String.valueOf("hive"))); + + // VarChar + Converter varcharConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, + PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector); + assertEquals("VarCharConverter", new HiveVarchar("TRUE", -1), varcharConverter + .convert(Boolean.valueOf(true))); + assertEquals("VarCharConverter", new HiveVarchar("FALSE", -1), varcharConverter + .convert(Boolean.valueOf(false))); + + varcharConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, + PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector); + assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("TRUE", -1)), varcharConverter + .convert(Boolean.valueOf(true))); + assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("FALSE", -1)), varcharConverter + .convert(Boolean.valueOf(false))); + + varcharConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaIntObjectInspector, + PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector); + assertEquals("VarCharConverter", new HiveVarchar("0", -1), varcharConverter + .convert(Integer.valueOf(0))); + assertEquals("VarCharConverter", new HiveVarchar("1", -1), varcharConverter + .convert(Integer.valueOf(1))); + + varcharConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaIntObjectInspector, + PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector); + assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("0", -1)), varcharConverter + .convert(Integer.valueOf(0))); + assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("1", -1)), varcharConverter + .convert(Integer.valueOf(1))); + + varcharConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector); + assertEquals("VarCharConverter", new HiveVarchar("hive", -1), varcharConverter + .convert(String.valueOf("hive"))); + + varcharConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector); + assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("hive", -1)), varcharConverter + .convert(String.valueOf("hive"))); + // Text Converter textConverter = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaIntObjectInspector, http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java ---------------------------------------------------------------------- diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java index 7d87666..29906c9 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java @@ -24,9 +24,13 @@ import java.util.List; import junit.framework.TestCase; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveCharWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; @@ -118,8 +122,14 @@ public class TestStandardObjectInspectors extends TestCase { .getClass()); } - assertEquals(PrimitiveObjectInspectorUtils + if (javaClass == HiveVarchar.class) { + assertEquals("varchar(65535)", oi1.getTypeName()); + } else if (javaClass == HiveChar.class) { + assertEquals("char(255)", oi1.getTypeName()); + } else { + assertEquals(PrimitiveObjectInspectorUtils .getTypeNameFromPrimitiveJava(javaClass), oi1.getTypeName()); + } } catch (Throwable e) { e.printStackTrace(); throw e; @@ -143,6 +153,8 @@ public class TestStandardObjectInspectors extends TestCase { doTestJavaPrimitiveObjectInspector(DoubleWritable.class, Double.class, (double) 1); doTestJavaPrimitiveObjectInspector(Text.class, String.class, "a"); + doTestJavaPrimitiveObjectInspector(HiveVarcharWritable.class, HiveVarchar.class, "a"); + doTestJavaPrimitiveObjectInspector(HiveCharWritable.class, HiveChar.class, "a"); doTestJavaPrimitiveObjectInspector(BytesWritable.class, byte[].class, new byte[]{'3'}); } catch (Throwable e) {
