DRILL-718: Return complex types as JSON string.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ea72a380 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ea72a380 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ea72a380 Branch: refs/heads/master Commit: ea72a380e875aaaa126510116742be659016d98f Parents: 4b57652 Author: Mehant Baid <[email protected]> Authored: Tue May 13 15:07:54 2014 -0700 Committer: Aditya Kishore <[email protected]> Committed: Tue May 13 16:06:03 2014 -0700 ---------------------------------------------------------------------- .../src/main/codegen/includes/vv_imports.ftl | 1 + .../codegen/templates/RepeatedValueVectors.java | 2 +- .../src/main/codegen/templates/TypeHelper.java | 5 ++ .../drill/exec/util/JsonStringArrayList.java | 46 ++++++++++++++++++ .../drill/exec/util/JsonStringHashMap.java | 50 ++++++++++++++++++++ .../apache/drill/exec/util/TextSerializer.java | 38 +++++++++++++++ .../drill/exec/vector/complex/MapVector.java | 3 +- .../exec/vector/complex/RepeatedListVector.java | 3 +- .../exec/vector/complex/RepeatedMapVector.java | 3 +- .../exec/vector/complex/fn/JsonReader.java | 2 +- .../exec/store/json/JsonRecordReader2Test.java | 2 + 11 files changed, 150 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/includes/vv_imports.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl index 71404d3..114ecbe 100644 --- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl +++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl @@ -38,6 +38,7 @@ import org.apache.drill.exec.vector.complex.impl.*; import org.apache.drill.exec.vector.complex.writer.*; import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; +import org.apache.drill.exec.util.JsonStringArrayList; import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java index 537c7fd..cac87c5 100644 --- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java @@ -296,7 +296,7 @@ package org.apache.drill.exec.vector; } public List<${friendlyType}> getObject(int index) { - List<${friendlyType}> vals = Lists.newArrayList(); + List<${friendlyType}> vals = new JsonStringArrayList(); int start = offsets.getAccessor().get(index) - sliceOffset; int end = offsets.getAccessor().get(index+1) - sliceOffset; for(int i = start; i < end; i++){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/TypeHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java index 4755e92..f2d66d2 100644 --- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java +++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java @@ -66,7 +66,12 @@ public class TypeHelper { } </#list> </#list> + case MAP: + case REPEATMAP: + case LIST: + return new GenericAccessor(vector); } + throw new UnsupportedOperationException(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java new file mode 100644 index 0000000..bc378a1 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.apache.hadoop.io.Text; + +import java.util.ArrayList; + +public class JsonStringArrayList extends ArrayList { + + private static ObjectMapper mapper; + + static { + mapper = new ObjectMapper(); + SimpleModule serializer = new SimpleModule("TextSerializer") + .addSerializer(Text.class, new TextSerializer()); + mapper.registerModule(serializer); + } + + @Override + public final String toString() { + try { + return mapper.writeValueAsString(this); + } catch(JsonProcessingException e) { + throw new IllegalStateException("Cannot serialize array list to JSON string", e); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java new file mode 100644 index 0000000..124252a --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.apache.hadoop.io.Text; + +import java.util.HashMap; + +/* + * Simple class that extends the regular java.util.HashMap but overrides the + * toString() method of the HashMap class to produce a JSON string instead + */ +public class JsonStringHashMap extends HashMap { + + private static ObjectMapper mapper; + + static { + mapper = new ObjectMapper(); + SimpleModule serializer = new SimpleModule("TextSerializer") + .addSerializer(Text.class, new TextSerializer()); + mapper.registerModule(serializer); + } + + @Override + public final String toString() { + try { + return mapper.writeValueAsString(this); + } catch(JsonProcessingException e) { + throw new IllegalStateException("Cannot serialize hash map to JSON string", e); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java new file mode 100644 index 0000000..3070592 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.util; + +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import org.apache.hadoop.io.Text; + +import java.io.IOException; + +public class TextSerializer extends StdSerializer<Text> { + + public TextSerializer() { + super(Text.class); + } + + @Override + public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonGenerationException { + jsonGenerator.writeString(text.toString()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java index 91c0be5..9b2e119 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java @@ -36,6 +36,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; import org.apache.drill.exec.proto.UserBitShared.SerializedField; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.TransferPair; +import org.apache.drill.exec.util.JsonStringHashMap; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.complex.RepeatedMapVector.MapSingleCopier; import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl; @@ -316,7 +317,7 @@ public class MapVector extends AbstractContainerVector { @Override public Object getObject(int index) { - Map<String, Object> vv = Maps.newHashMap(); + Map<String, Object> vv = new JsonStringHashMap(); for(Map.Entry<String, ValueVector> e : vectors.entrySet()){ ValueVector v = e.getValue(); String k = e.getKey(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java index 93930b5..88d858a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java @@ -37,6 +37,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; import org.apache.drill.exec.proto.UserBitShared.SerializedField; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.TransferPair; +import org.apache.drill.exec.util.JsonStringArrayList; import org.apache.drill.exec.vector.RepeatedFixedWidthVector; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; @@ -155,7 +156,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea @Override public Object getObject(int index) { - List<Object> l = Lists.newArrayList(); + List<Object> l = new JsonStringArrayList(); int end = offsets.getAccessor().get(index+1); for(int i = offsets.getAccessor().get(index); i < end; i++){ l.add(vector.getAccessor().getObject(i)); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java index 2492cc8..22471f0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java @@ -36,6 +36,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; import org.apache.drill.exec.proto.UserBitShared.SerializedField; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.TransferPair; +import org.apache.drill.exec.util.JsonStringArrayList; import org.apache.drill.exec.vector.RepeatedFixedWidthVector; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; @@ -348,7 +349,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat @Override public Object getObject(int index) { - List<Object> l = Lists.newArrayList(); + List<Object> l = new JsonStringArrayList(); int end = offsets.getAccessor().get(index+1); for(int i = offsets.getAccessor().get(index); i < end; i++){ Map<String, Object> vv = Maps.newHashMap(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java index 761bc79..e5d4b5b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java @@ -221,7 +221,7 @@ public class JsonReader { vh.start = 0; vh.end = b.length; list.varChar().write(vh); - + break; default: throw new IllegalStateException("Unexpected token " + parser.getCurrentToken()); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java index 1372219..ad46326 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java @@ -27,6 +27,8 @@ public class JsonRecordReader2Test extends BaseTestQuery{ public void testComplexJsonInput() throws Exception{ // test("select z[0]['orange'] from cp.`jsoninput/input2.json` limit 10"); test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], z[0]['orange'], z[1]['pink'] from cp.`jsoninput/input2.json` limit 10"); + test("select x from cp.`jsoninput/input2.json`"); + // test("select z[0] from cp.`jsoninput/input2.json` limit 10"); }
