http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java index f7adb39,d82e93c..e1b6dd8 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java @@@ -17,10 -17,9 +17,9 @@@ import java.io.IOException import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.exec.Utilities; - import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface; import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport; import org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper; -import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.RecordReader;
http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java index 3cd48f8,3261e4b..cb25a4f --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java @@@ -13,11 -13,12 +13,11 @@@ */ package org.apache.hadoop.hive.ql.io.parquet.convert; -import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport; -import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; - import parquet.io.api.GroupConverter; - import parquet.io.api.RecordMaterializer; - import parquet.schema.GroupType; - import parquet.schema.MessageType; + import org.apache.parquet.io.api.GroupConverter; + import org.apache.parquet.io.api.RecordMaterializer; + import org.apache.parquet.schema.GroupType; -import org.apache.parquet.schema.MessageTypeParser; ++import org.apache.parquet.schema.MessageType; import java.util.Map; http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java index 7915111,06f3d32..99d3643 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java @@@ -20,13 -20,13 +20,15 @@@ package org.apache.hadoop.hive.ql.io.pa import com.google.common.base.Preconditions; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; - import parquet.io.api.Converter; - import parquet.schema.GroupType; - import parquet.schema.Type; + import org.apache.hadoop.io.ArrayWritable; + import org.apache.hadoop.io.Writable; + import org.apache.parquet.io.api.Converter; + import org.apache.parquet.schema.GroupType; + import org.apache.parquet.schema.Type; public class HiveCollectionConverter extends HiveGroupConverter { private final GroupType collectionType; http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java index c22c045,b1ca85a..30001cb --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java @@@ -13,14 -13,14 +13,13 @@@ */ package org.apache.hadoop.hive.ql.io.parquet.convert; --import org.apache.hadoop.io.Writable; - import parquet.io.api.Converter; - import parquet.io.api.GroupConverter; - import parquet.io.api.PrimitiveConverter; - import parquet.schema.GroupType; - import parquet.schema.OriginalType; - import parquet.schema.PrimitiveType; - import parquet.schema.Type; + import org.apache.parquet.io.api.Converter; + import org.apache.parquet.io.api.GroupConverter; + import org.apache.parquet.io.api.PrimitiveConverter; + import org.apache.parquet.schema.GroupType; + import org.apache.parquet.schema.OriginalType; + import org.apache.parquet.schema.PrimitiveType; + import org.apache.parquet.schema.Type; import java.util.Map; http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java index 4d06c36,9c35a9f..bd706fa --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java @@@ -18,10 -17,11 +18,10 @@@ import java.util.Arrays import java.util.List; import java.util.Map; -import org.apache.hadoop.io.ArrayWritable; -import org.apache.hadoop.io.Writable; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; - import parquet.io.api.Converter; - import parquet.schema.GroupType; - import parquet.schema.Type; + import org.apache.parquet.io.api.Converter; + import org.apache.parquet.schema.GroupType; + import org.apache.parquet.schema.Type; /** * http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java index c84caec,c0af291..3b36eaf --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java @@@ -22,13 -22,15 +22,13 @@@ import java.util.ArrayList import java.util.List; import java.util.Map; -import org.apache.hadoop.io.ArrayWritable; -import org.apache.hadoop.io.Writable; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; - import parquet.column.Dictionary; - import parquet.io.api.Binary; - import parquet.io.api.Converter; - import parquet.io.api.PrimitiveConverter; - import parquet.schema.GroupType; - import parquet.schema.PrimitiveType; + import org.apache.parquet.column.Dictionary; + import org.apache.parquet.io.api.Binary; + import org.apache.parquet.io.api.Converter; + import org.apache.parquet.io.api.PrimitiveConverter; + import org.apache.parquet.schema.GroupType; + import org.apache.parquet.schema.PrimitiveType; /** * Converters for repeated fields need to know when the parent field starts and http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java index a5856bf,8b8b92d..c5ad201 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java @@@ -29,17 -29,17 +29,17 @@@ import org.apache.hadoop.hive.serde2.ob import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; import org.apache.hadoop.util.StringUtils; - import parquet.hadoop.api.InitContext; - import parquet.hadoop.api.ReadSupport; - import parquet.io.api.RecordMaterializer; - import parquet.schema.GroupType; - import parquet.schema.MessageType; - import parquet.schema.Type; - import parquet.schema.Types; - import parquet.schema.PrimitiveType.PrimitiveTypeName; + import org.apache.parquet.hadoop.api.InitContext; + import org.apache.parquet.hadoop.api.ReadSupport; + import org.apache.parquet.io.api.RecordMaterializer; + import org.apache.parquet.schema.GroupType; + import org.apache.parquet.schema.MessageType; + import org.apache.parquet.schema.Type; + import org.apache.parquet.schema.Types; + import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; /** * @@@ -246,9 -245,9 +246,9 @@@ public class DataWritableReadSupport ex * @return Record Materialize for Hive */ @Override - public RecordMaterializer<ArrayWritable> prepareForRead(final Configuration configuration, + public RecordMaterializer<ObjectArrayWritable> prepareForRead(final Configuration configuration, final Map<String, String> keyValueMetaData, final MessageType fileSchema, - final parquet.hadoop.api.ReadSupport.ReadContext readContext) { + final org.apache.parquet.hadoop.api.ReadSupport.ReadContext readContext) { final Map<String, String> metadata = readContext.getReadSupportMetadata(); if (metadata == null) { throw new IllegalStateException("ReadContext not initialized properly. " + http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java index dc4f896,e1bf8e2..58eabf0 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java @@@ -34,10 -33,11 +34,10 @@@ import org.apache.hadoop.hive.serde2.ty import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; - import parquet.hadoop.ParquetOutputFormat; - import parquet.hadoop.ParquetWriter; + import org.apache.parquet.hadoop.ParquetOutputFormat; + import org.apache.parquet.hadoop.ParquetWriter; /** * http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java ---------------------------------------------------------------------- diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java index 17d8feb,4b69642..e3e875e --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java @@@ -20,18 -20,22 +20,18 @@@ package org.apache.hadoop.hive.ql.io.pa import java.util.List; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.io.ArrayWritable; -import org.apache.hadoop.io.FloatWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; import org.junit.Assert; import org.junit.Test; - import parquet.io.api.RecordConsumer; - import parquet.schema.MessageType; - import parquet.schema.Types; + import org.apache.parquet.io.api.RecordConsumer; + import org.apache.parquet.schema.MessageType; + import org.apache.parquet.schema.Types; - import static parquet.schema.OriginalType.LIST; - import static parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE; - import static parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT; - import static parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; - import static parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; + import static org.apache.parquet.schema.OriginalType.LIST; + import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE; + import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT; + import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; + import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; public class TestArrayCompatibility extends AbstractTestParquetDirect { http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java ---------------------------------------------------------------------- diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java index 5f586e4,7049139..71f4acf --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestDataWritableWriter.java @@@ -353,21 -358,19 +353,19 @@@ public class TestDataWritableWriter + " }\n" + "}\n"; - ArrayWritable hiveRecord = createGroup( + ObjectArrayWritable hiveRecord = createGroup( createGroup( createArray( - createArray( - createString("key1"), - createInt(1) - ), - createArray( - createString("key2"), - createInt(2) - ), - createArray( - createString("key3"), - createNull() - ) + createString("key1"), + createInt(1) + ), + createArray( + createString("key2"), + createInt(2) + ), + createArray( + createString("key3"), + createNull() ) ) ); @@@ -424,15 -427,11 +422,11 @@@ + " }\n" + "}\n"; - ArrayWritable hiveRecord = createGroup( - createArray( + ObjectArrayWritable hiveRecord = createGroup( + createGroup( createArray( - createGroup( - createArray( - createInt(1), - createInt(2) - ) - ) + createInt(1), + createInt(2) ) ) ); http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java ---------------------------------------------------------------------- diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java index 1b98cd3,518eefd..fdf4b0f --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetInputFormat.java @@@ -15,10 -15,10 +15,10 @@@ package org.apache.hadoop.hive.ql.io.pa import static org.mockito.Mockito.mock; -import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; import org.junit.Test; - import parquet.hadoop.ParquetInputFormat; + import org.apache.parquet.hadoop.ParquetInputFormat; public class TestMapredParquetInputFormat { @Test http://git-wip-us.apache.org/repos/asf/hive/blob/68fd78dd/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetRowGroupFilter.java ----------------------------------------------------------------------