Author: daijy Date: Thu Oct 30 18:27:59 2014 New Revision: 1635577 URL: http://svn.apache.org/r1635577 Log: HIVE-7282: HCatLoader fail to load Orc map with null key (Daniel Dai)
Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java?rev=1635577&r1=1635576&r2=1635577&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java (original) +++ hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java Thu Oct 30 18:27:59 2014 @@ -23,7 +23,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.TreeMap; +import java.util.HashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.serdeConstants; @@ -212,7 +212,7 @@ public class HCatRecordSerDe implements private static Map<?, ?> serializeMap(Object f, MapObjectInspector moi) throws SerDeException { ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); - Map<Object, Object> m = new TreeMap<Object, Object>(); + Map<Object, Object> m = new HashMap<Object, Object>(); Map<?, ?> readMap = moi.getMap(f); if (readMap == null) { Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java?rev=1635577&r1=1635576&r2=1635577&view=diff ============================================================================== --- hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java (original) +++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java Thu Oct 30 18:27:59 2014 @@ -480,7 +480,9 @@ class PigHCatUtil { Map<String, Object> result = new HashMap<String, Object>(); for (Entry<?, ?> entry : map.entrySet()) { // since map key for Pig has to be Strings - result.put(entry.getKey().toString(), extractPigObject(entry.getValue(), hfs.getMapValueSchema().get(0))); + if (entry.getKey()!=null) { + result.put(entry.getKey().toString(), extractPigObject(entry.getValue(), hfs.getMapValueSchema().get(0))); + } } return result; } Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1635577&r1=1635576&r2=1635577&view=diff ============================================================================== --- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java (original) +++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java Thu Oct 30 18:27:59 2014 @@ -18,8 +18,6 @@ */ package org.apache.hive.hcatalog.pig; -import com.google.common.collect.ImmutableSet; - import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -52,7 +50,6 @@ import org.apache.pig.impl.logicalLayer. import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -79,11 +76,13 @@ public class TestHCatLoaderComplexSchema add("testSyntheticComplexSchema"); add("testTupleInBagInTupleInBag"); add("testMapWithComplexData"); + add("testMapNullKey"); }}); put(IOConstants.PARQUETFILE, new HashSet<String>() {{ add("testSyntheticComplexSchema"); add("testTupleInBagInTupleInBag"); add("testMapWithComplexData"); + add("testMapNullKey"); }}); }}; @@ -223,6 +222,10 @@ public class TestHCatLoaderComplexSchema private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, boolean provideSchemaToStorer) throws IOException, CommandNeedRetryException, ExecException, FrontendException { + verifyWriteRead(tablename, pigSchema, tableSchema, data, data, provideSchemaToStorer); + } + private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, List<Tuple> result, boolean provideSchemaToStorer) + throws IOException, CommandNeedRetryException, ExecException, FrontendException { MockLoader.setData(tablename + "Input", data); try { createTable(tablename, tableSchema); @@ -244,7 +247,7 @@ public class TestHCatLoaderComplexSchema Iterator<Tuple> it = server.openIterator("X"); int i = 0; while (it.hasNext()) { - Tuple input = data.get(i++); + Tuple input = result.get(i++); Tuple output = it.next(); compareTuples(input, output); LOG.info("tuple : {} ", output); @@ -354,4 +357,40 @@ public class TestHCatLoaderComplexSchema verifyWriteRead("testMapWithComplexData", pigSchema, tableSchema, data, true); verifyWriteRead("testMapWithComplexData2", pigSchema, tableSchema, data, false); } + + /** + * artificially complex nested schema to test nested schema conversion + * @throws Exception + */ + @Test + public void testMapNullKey() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); + String pigSchema = "m:map[]"; + + String tableSchema = "m map<string, string>"; + + List<Tuple> data = new ArrayList<Tuple>(); + Tuple t = t( + new HashMap<String, String>() { + { + put("ac test1", "test 1"); + put("ac test2", "test 2"); + put(null, "test 3"); + }; + }); + data.add(t); + + List<Tuple> result = new ArrayList<Tuple>(); + t = t( + new HashMap<String, String>() { + { + put("ac test1", "test 1"); + put("ac test2", "test 2"); + }; + }); + result.add(t); + + verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, true); + verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, false); + } }