HIVE-10437 : NullPointerException on queries where map/reduce is not involved on tables with partitions (Ashutosh Chauhan via Gunther Hagleitner)
Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e6849017 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e6849017 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e6849017 Branch: refs/heads/branch-1.2 Commit: e6849017df210697ce3a2067e072eb074a6a18f5 Parents: cef4ce0 Author: Ashutosh Chauhan <hashut...@apache.org> Authored: Mon Apr 27 13:48:00 2015 -0700 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Wed Apr 29 16:33:38 2015 -0700 ---------------------------------------------------------------------- .../java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java | 9 +++++++-- .../main/java/org/apache/hive/hcatalog/data/JsonSerDe.java | 7 ++++++- .../java/org/apache/hadoop/hive/ql/exec/FetchOperator.java | 8 +++++++- 3 files changed, 20 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e6849017/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java index 722e05d..81c7943 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; @@ -51,6 +52,10 @@ import org.slf4j.LoggerFactory; /** * SerDe class for serializing to and from HCatRecord */ + +@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, + serdeConstants.LIST_COLUMN_TYPES}) + public class HCatRecordSerDe implements SerDe { private static final Logger LOG = LoggerFactory.getLogger(HCatRecordSerDe.class); @@ -124,7 +129,7 @@ public class HCatRecordSerDe implements SerDe { throw new SerDeException(getClass().getName() + ": expects HCatRecord!"); } - return (HCatRecord) data; + return data; } /** @@ -302,7 +307,7 @@ public class HCatRecordSerDe implements SerDe { */ @Override public ObjectInspector getObjectInspector() throws SerDeException { - return (ObjectInspector) cachedObjectInspector; + return cachedObjectInspector; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/e6849017/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java index 6b68323..9b325b6 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; @@ -84,6 +85,10 @@ import org.codehaus.jackson.JsonToken; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, + serdeConstants.LIST_COLUMN_TYPES, + serdeConstants.TIMESTAMP_FORMATS}) + public class JsonSerDe implements SerDe { private static final Logger LOG = LoggerFactory.getLogger(JsonSerDe.class); @@ -497,7 +502,7 @@ public class JsonSerDe implements SerDe { break; } case STRING: { - String s = + String s = SerDeUtils.escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)); appendWithQuotes(sb, s); break; http://git-wip-us.apache.org/repos/asf/hive/blob/e6849017/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index 92968b9..258d28e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -604,7 +604,13 @@ public class FetchOperator implements Serializable { // if table and all partitions have the same schema and serde, no need to convert private boolean needConversion(TableDesc tableDesc, List<PartitionDesc> partDescs) { Class<?> tableSerDe = tableDesc.getDeserializerClass(); - String[] schemaProps = AnnotationUtils.getAnnotation(tableSerDe, SerDeSpec.class).schemaProps(); + SerDeSpec spec = AnnotationUtils.getAnnotation(tableSerDe, SerDeSpec.class); + if (null == spec) { + // Serde may not have this optional annotation defined in which case be conservative + // and say conversion is needed. + return true; + } + String[] schemaProps = spec.schemaProps(); Properties tableProps = tableDesc.getProperties(); for (PartitionDesc partitionDesc : partDescs) { if (!tableSerDe.getName().equals(partitionDesc.getDeserializerClassName())) {
