HIVE-12653 : The property "serialization.encoding" in the class "org.apache.hadoop.hive.contrib.serde2.MultiDelimitSerDe" does not work (yangfeng via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2449d1df Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2449d1df Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2449d1df Branch: refs/heads/llap Commit: 2449d1dfe9429363a9458d2004ec2405f5aa9035 Parents: e384b2b Author: yangfang <yang.fa...@zte.com.cn> Authored: Mon Dec 14 03:13:00 2015 -0800 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Fri Mar 25 07:43:05 2016 -0700 ---------------------------------------------------------------------- .../hive/contrib/serde2/MultiDelimitSerDe.java | 23 +++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2449d1df/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java ---------------------------------------------------------------------- diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java index 9a162d5..296c449 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java @@ -63,7 +63,7 @@ import org.apache.hadoop.io.Writable; serdeConstants.SERIALIZATION_ENCODING, LazySerDeParameters.SERIALIZATION_EXTEND_NESTING_LEVELS, LazySerDeParameters.SERIALIZATION_EXTEND_ADDITIONAL_NESTING_LEVELS}) -public class MultiDelimitSerDe extends AbstractSerDe { +public class MultiDelimitSerDe extends AbstractEncodingAwareSerDe { private static final byte[] DEFAULT_SEPARATORS = {(byte) 1, (byte) 2, (byte) 3}; // Due to HIVE-6404, define our own constant @@ -94,6 +94,7 @@ public class MultiDelimitSerDe extends AbstractSerDe { @Override public void initialize(Configuration conf, Properties tbl) throws SerDeException { // get the SerDe parameters + super.initialize(conf, tbl); serdeParams = new LazySerDeParameters(conf, tbl, getClass().getName()); fieldDelimited = tbl.getProperty(serdeConstants.FIELD_DELIM); @@ -134,8 +135,9 @@ public class MultiDelimitSerDe extends AbstractSerDe { return Text.class; } - @Override - public Object deserialize(Writable blob) throws SerDeException { + + @Override + public Object doDeserialize(Writable blob) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } @@ -159,8 +161,9 @@ public class MultiDelimitSerDe extends AbstractSerDe { return cachedLazyStruct; } - @Override - public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { + @Override + public Writable doSerialize(Object obj, ObjectInspector objInspector) + throws SerDeException { StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); List<Object> list = soi.getStructFieldsDataAsList(obj); @@ -286,6 +289,16 @@ public class MultiDelimitSerDe extends AbstractSerDe { throw new RuntimeException("Unknown category type: "+ objInspector.getCategory()); } + protected Text transformFromUTF8(Writable blob) { + Text text = (Text)blob; + return SerDeUtils.transformTextFromUTF8(text, this.charset); + } + + protected Text transformToUTF8(Writable blob) { + Text text = (Text) blob; + return SerDeUtils.transformTextToUTF8(text, this.charset); + } + @Override public SerDeStats getSerDeStats() { // no support for statistics