DRILL-406: Move JSON integer range to use BigInt instead of Int
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/11f30a08 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/11f30a08 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/11f30a08 Branch: refs/heads/master Commit: 11f30a084954fda5d0f82570468c5a1056402137 Parents: db00e97 Author: Steven Phillips <[email protected]> Authored: Mon Mar 17 09:09:59 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Mon Mar 17 09:09:59 2014 -0700 ---------------------------------------------------------------------- .../exec/schema/json/jackson/JacksonHelper.java | 8 +-- .../exec/store/easy/json/JSONRecordReader.java | 25 +++---- .../exec/store/json/JSONRecordReaderTest.java | 76 ++++++++++---------- 3 files changed, 50 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/11f30a08/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java index 51f273f..1e54336 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java @@ -33,8 +33,8 @@ public class JacksonHelper { public static final MajorType REPEATED_BOOLEAN_TYPE = Types.repeated(MinorType.BIT); public static final MajorType ARRAY_TYPE = Types.repeated(MinorType.LATE); public static final MajorType MAP_TYPE = Types.repeated(MinorType.MAP); - public static final MajorType INT_TYPE = Types.optional(MinorType.INT); - public static final MajorType REPEATED_INT_TYPE = Types.repeated(MinorType.INT); + public static final MajorType INT_TYPE = Types.optional(MinorType.BIGINT); + public static final MajorType REPEATED_INT_TYPE = Types.repeated(MinorType.BIGINT); public static final MajorType FLOAT_TYPE = Types.optional(MinorType.FLOAT4); public static final MajorType REPEATED_FLOAT_TYPE = Types.repeated(MinorType.FLOAT4); public static final MajorType NULL_TYPE = Types.optional(MinorType.LATE); @@ -82,8 +82,8 @@ public class JacksonHelper { public static Object getValueFromFieldType(JsonParser parser, MinorType fieldType) throws IOException { switch (fieldType) { - case INT: - return parser.getIntValue(); + case BIGINT: + return parser.getLongValue(); case VARCHAR: return parser.getValueAsString(); case FLOAT4: http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/11f30a08/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java index 67e8b3f..80f8678 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java @@ -52,16 +52,7 @@ import org.apache.drill.exec.schema.SchemaIdGenerator; import org.apache.drill.exec.schema.json.jackson.JacksonHelper; import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.store.VectorHolder; -import org.apache.drill.exec.vector.AllocationHelper; -import org.apache.drill.exec.vector.NullableBitVector; -import org.apache.drill.exec.vector.NullableFloat4Vector; -import org.apache.drill.exec.vector.NullableIntVector; -import org.apache.drill.exec.vector.NullableVarCharVector; -import org.apache.drill.exec.vector.RepeatedBitVector; -import org.apache.drill.exec.vector.RepeatedFloat4Vector; -import org.apache.drill.exec.vector.RepeatedIntVector; -import org.apache.drill.exec.vector.RepeatedVarCharVector; -import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.*; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -412,23 +403,23 @@ public class JSONRecordReader implements RecordReader { private static <T> boolean addValueToVector(int index, VectorHolder holder, T val, MinorType minorType, int groupCount) { switch (minorType) { - case INT: { + case BIGINT: { holder.incAndCheckLength(NullableIntHolder.WIDTH * 8 + 1); if (groupCount == 0) { if (val != null) { - NullableIntVector int4 = (NullableIntVector) holder.getValueVector(); - NullableIntVector.Mutator m = int4.getMutator(); - m.set(index, (Integer) val); + NullableBigIntVector int4 = (NullableBigIntVector) holder.getValueVector(); + NullableBigIntVector.Mutator m = int4.getMutator(); + m.set(index, (Long) val); } } else { if (val == null) { throw new UnsupportedOperationException("Nullable repeated int is not supported."); } - RepeatedIntVector repeatedInt4 = (RepeatedIntVector) holder.getValueVector(); - RepeatedIntVector.Mutator m = repeatedInt4.getMutator(); + RepeatedBigIntVector repeatedInt4 = (RepeatedBigIntVector) holder.getValueVector(); + RepeatedBigIntVector.Mutator m = repeatedInt4.getMutator(); holder.setGroupCount(index); - m.add(index, (Integer) val); + m.add(index, (Long) val); } return holder.hasEnoughSpace(NullableIntHolder.WIDTH * 8 + 1); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/11f30a08/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java index 41960b6..bfeb892 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java @@ -99,7 +99,7 @@ public class JSONRecordReaderTest { SchemaDefProtos.FieldDef def = metadata.getDef(); assertEquals(expectedMinorType, def.getMajorType().getMinorType()); String[] parts = name.split("\\."); - int expected = parts.length; + long expected = parts.length; assertEquals(expected, def.getNameList().size()); for(int i = 0; i < parts.length; ++i) { assertEquals(parts[i], def.getName(i).getName()); @@ -146,10 +146,10 @@ public class JSONRecordReaderTest { jr.setup(mutator); assertEquals(2, jr.next()); assertEquals(3, addFields.size()); - assertField(addFields.get(0), 0, MinorType.INT, 123, "test"); + assertField(addFields.get(0), 0, MinorType.BIGINT, 123L, "test"); assertField(addFields.get(1), 0, MinorType.BIT, true, "b"); assertField(addFields.get(2), 0, MinorType.VARCHAR, "hi!", "c"); - assertField(addFields.get(0), 1, MinorType.INT, 1234, "test"); + assertField(addFields.get(0), 1, MinorType.BIGINT, 1234L, "test"); assertField(addFields.get(1), 1, MinorType.BIT, false, "b"); assertField(addFields.get(2), 1, MinorType.VARCHAR, "drill!", "c"); @@ -177,22 +177,22 @@ public class JSONRecordReaderTest { jr.setup(mutator); assertEquals(3, jr.next()); assertEquals(7, addFields.size()); - assertField(addFields.get(0), 0, MinorType.INT, 123, "test"); - assertField(addFields.get(1), 0, MinorType.INT, 1, "b"); + assertField(addFields.get(0), 0, MinorType.BIGINT, 123L, "test"); + assertField(addFields.get(1), 0, MinorType.BIGINT, 1L, "b"); assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 2.15, "c"); assertField(addFields.get(3), 0, MinorType.BIT, true, "bool"); assertField(addFields.get(4), 0, MinorType.VARCHAR, "test1", "str1"); - assertField(addFields.get(0), 1, MinorType.INT, 1234, "test"); - assertField(addFields.get(1), 1, MinorType.INT, 3, "b"); + assertField(addFields.get(0), 1, MinorType.BIGINT, 1234L, "test"); + assertField(addFields.get(1), 1, MinorType.BIGINT, 3L, "b"); assertField(addFields.get(3), 1, MinorType.BIT, false, "bool"); assertField(addFields.get(4), 1, MinorType.VARCHAR, "test2", "str1"); - assertField(addFields.get(5), 1, MinorType.INT, 4, "d"); + assertField(addFields.get(5), 1, MinorType.BIGINT, 4L, "d"); - assertField(addFields.get(0), 2, MinorType.INT, 12345, "test"); + assertField(addFields.get(0), 2, MinorType.BIGINT, 12345L, "test"); assertField(addFields.get(2), 2, MinorType.FLOAT4, (float) 5.16, "c"); assertField(addFields.get(3), 2, MinorType.BIT, true, "bool"); - assertField(addFields.get(5), 2, MinorType.INT, 6, "d"); + assertField(addFields.get(5), 2, MinorType.BIGINT, 6L, "d"); assertField(addFields.get(6), 2, MinorType.VARCHAR, "test3", "str2"); assertTrue(mutator.getRemovedFields().isEmpty()); assertEquals(0, jr.next()); @@ -218,16 +218,16 @@ public class JSONRecordReaderTest { jr.setup(mutator); assertEquals(1, jr.next()); - assertField(addFields.get(0), 0, MinorType.INT, 123, "test"); + assertField(addFields.get(0), 0, MinorType.BIGINT, 123L, "test"); assertTrue(removedFields.isEmpty()); assertEquals(addFields.size(), 1); assertEquals(1, jr.next()); - assertField(addFields.get(0), 0, MinorType.INT, 1234, "test"); + assertField(addFields.get(0), 0, MinorType.BIGINT, 1234L, "test"); assertEquals(addFields.size(), 1); assertTrue(removedFields.isEmpty()); removedFields.clear(); assertEquals(1, jr.next()); - assertField(addFields.get(0), 0, MinorType.INT, 12345, "test"); + assertField(addFields.get(0), 0, MinorType.BIGINT, 12345L, "test"); assertEquals(addFields.size(), 1); assertTrue(removedFields.isEmpty()); assertEquals(0, jr.next()); @@ -254,27 +254,27 @@ public class JSONRecordReaderTest { jr.setup(mutator); assertEquals(1, jr.next()); assertEquals(5, addFields.size()); - assertField(addFields.get(0), 0, MinorType.INT, 123, "test"); - assertField(addFields.get(1), 0, MinorType.INT, 1, "b"); + assertField(addFields.get(0), 0, MinorType.BIGINT, 123L, "test"); + assertField(addFields.get(1), 0, MinorType.BIGINT, 1L, "b"); assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 2.15, "c"); assertField(addFields.get(3), 0, MinorType.BIT, true, "bool"); assertField(addFields.get(4), 0, MinorType.VARCHAR, "test1", "str1"); assertTrue(removedFields.isEmpty()); assertEquals(1, jr.next()); assertEquals(6, addFields.size()); - assertField(addFields.get(0), 0, MinorType.INT, 1234, "test"); - assertField(addFields.get(1), 0, MinorType.INT, 3, "b"); + assertField(addFields.get(0), 0, MinorType.BIGINT, 1234L, "test"); + assertField(addFields.get(1), 0, MinorType.BIGINT, 3L, "b"); assertField(addFields.get(3), 0, MinorType.BIT, false, "bool"); assertField(addFields.get(4), 0, MinorType.VARCHAR, "test2", "str1"); - assertField(addFields.get(5), 0, MinorType.INT, 4, "d"); + assertField(addFields.get(5), 0, MinorType.BIGINT, 4L, "d"); assertEquals(1, removedFields.size()); assertEquals("c", removedFields.get(0).getName()); removedFields.clear(); assertEquals(1, jr.next()); assertEquals(7, addFields.size()); // The reappearing of field 'c' is also included - assertField(addFields.get(0), 0, MinorType.INT, 12345, "test"); + assertField(addFields.get(0), 0, MinorType.BIGINT, 12345L, "test"); assertField(addFields.get(3), 0, MinorType.BIT, true, "bool"); - assertField(addFields.get(5), 0, MinorType.INT, 6, "d"); + assertField(addFields.get(5), 0, MinorType.BIGINT, 6L, "d"); assertField(addFields.get(2), 0, MinorType.FLOAT4, (float) 5.16, "c"); assertField(addFields.get(6), 0, MinorType.VARCHAR, "test3", "str2"); assertEquals(2, removedFields.size()); @@ -311,10 +311,10 @@ public class JSONRecordReaderTest { jr.setup(mutator); assertEquals(2, jr.next()); assertEquals(3, addFields.size()); - assertField(addFields.get(0), 0, MinorType.INT, 123, "test"); + assertField(addFields.get(0), 0, MinorType.BIGINT, 123L, "test"); assertField(addFields.get(1), 0, MinorType.VARCHAR, "test", "a.b"); assertField(addFields.get(2), 0, MinorType.BIT, true, "a.a.d"); - assertField(addFields.get(0), 1, MinorType.INT, 1234, "test"); + assertField(addFields.get(0), 1, MinorType.BIGINT, 1234L, "test"); assertField(addFields.get(1), 1, MinorType.VARCHAR, "test2", "a.b"); assertField(addFields.get(2), 1, MinorType.BIT, false, "a.a.d"); @@ -340,15 +340,15 @@ public class JSONRecordReaderTest { jr.setup(mutator); assertEquals(2, jr.next()); assertEquals(7, addFields.size()); - assertField(addFields.get(0), 0, MinorType.INT, 123, "test"); - assertField(addFields.get(1), 0, MinorType.INT, Arrays.asList(1, 2, 3), "test2"); - assertField(addFields.get(2), 0, MinorType.INT, Arrays.asList(4, 5, 6), "test3.a"); - assertField(addFields.get(3), 0, MinorType.INT, Arrays.asList(7, 8, 9), "test3.b"); - assertField(addFields.get(4), 0, MinorType.INT, Arrays.asList(10, 11, 12), "test3.c.d"); + assertField(addFields.get(0), 0, MinorType.BIGINT, 123L, "test"); + assertField(addFields.get(1), 0, MinorType.BIGINT, Arrays.asList(1L, 2L, 3L), "test2"); + assertField(addFields.get(2), 0, MinorType.BIGINT, Arrays.asList(4L, 5L, 6L), "test3.a"); + assertField(addFields.get(3), 0, MinorType.BIGINT, Arrays.asList(7L, 8L, 9L), "test3.b"); + assertField(addFields.get(4), 0, MinorType.BIGINT, Arrays.asList(10L, 11L, 12L), "test3.c.d"); assertField(addFields.get(5), 0, MinorType.FLOAT4, Arrays.<Float>asList((float) 1.1, (float) 1.2, (float) 1.3), "testFloat"); assertField(addFields.get(6), 0, MinorType.VARCHAR, Arrays.asList("hello", "drill"), "testStr"); - assertField(addFields.get(1), 1, MinorType.INT, Arrays.asList(1, 2), "test2"); - assertField(addFields.get(2), 1, MinorType.INT, Arrays.asList(7, 7, 7, 8), "test3.a"); + assertField(addFields.get(1), 1, MinorType.BIGINT, Arrays.asList(1L, 2L), "test2"); + assertField(addFields.get(2), 1, MinorType.BIGINT, Arrays.asList(7L, 7L, 7L, 8L), "test3.a"); assertField(addFields.get(5), 1, MinorType.FLOAT4, Arrays.<Float>asList((float) 2.2, (float) 2.3,(float) 2.4), "testFloat"); assertEquals(0, jr.next()); @@ -373,15 +373,15 @@ public class JSONRecordReaderTest { jr.setup(mutator); assertEquals(9, jr.next()); assertEquals(1, addFields.size()); - assertField(addFields.get(0), 0, MinorType.INT, Arrays.<Integer>asList(), "test"); - assertField(addFields.get(0), 1, MinorType.INT, Arrays.asList(1, 2, 3), "test"); - assertField(addFields.get(0), 2, MinorType.INT, Arrays.<Integer>asList(), "test"); - assertField(addFields.get(0), 3, MinorType.INT, Arrays.<Integer>asList(), "test"); - assertField(addFields.get(0), 4, MinorType.INT, Arrays.asList(4, 5, 6), "test"); - assertField(addFields.get(0), 5, MinorType.INT, Arrays.<Integer>asList(), "test"); - assertField(addFields.get(0), 6, MinorType.INT, Arrays.<Integer>asList(), "test"); - assertField(addFields.get(0), 7, MinorType.INT, Arrays.asList(7, 8, 9), "test"); - assertField(addFields.get(0), 8, MinorType.INT, Arrays.<Integer>asList(), "test"); + assertField(addFields.get(0), 0, MinorType.BIGINT, Arrays.<Long>asList(), "test"); + assertField(addFields.get(0), 1, MinorType.BIGINT, Arrays.asList(1L, 2L, 3L), "test"); + assertField(addFields.get(0), 2, MinorType.BIGINT, Arrays.<Long>asList(), "test"); + assertField(addFields.get(0), 3, MinorType.BIGINT, Arrays.<Long>asList(), "test"); + assertField(addFields.get(0), 4, MinorType.BIGINT, Arrays.asList(4L, 5L, 6L), "test"); + assertField(addFields.get(0), 5, MinorType.BIGINT, Arrays.<Long>asList(), "test"); + assertField(addFields.get(0), 6, MinorType.BIGINT, Arrays.<Long>asList(), "test"); + assertField(addFields.get(0), 7, MinorType.BIGINT, Arrays.asList(7L, 8L, 9L), "test"); + assertField(addFields.get(0), 8, MinorType.BIGINT, Arrays.<Long>asList(), "test"); assertEquals(0, jr.next());
