DRILL-718: Return complex types as JSON string.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ea72a380
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ea72a380
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ea72a380
Branch: refs/heads/master
Commit: ea72a380e875aaaa126510116742be659016d98f
Parents: 4b57652
Author: Mehant Baid <[email protected]>
Authored: Tue May 13 15:07:54 2014 -0700
Committer: Aditya Kishore <[email protected]>
Committed: Tue May 13 16:06:03 2014 -0700
----------------------------------------------------------------------
.../src/main/codegen/includes/vv_imports.ftl | 1 +
.../codegen/templates/RepeatedValueVectors.java | 2 +-
.../src/main/codegen/templates/TypeHelper.java | 5 ++
.../drill/exec/util/JsonStringArrayList.java | 46 ++++++++++++++++++
.../drill/exec/util/JsonStringHashMap.java | 50 ++++++++++++++++++++
.../apache/drill/exec/util/TextSerializer.java | 38 +++++++++++++++
.../drill/exec/vector/complex/MapVector.java | 3 +-
.../exec/vector/complex/RepeatedListVector.java | 3 +-
.../exec/vector/complex/RepeatedMapVector.java | 3 +-
.../exec/vector/complex/fn/JsonReader.java | 2 +-
.../exec/store/json/JsonRecordReader2Test.java | 2 +
11 files changed, 150 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
index 71404d3..114ecbe 100644
--- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -38,6 +38,7 @@ import org.apache.drill.exec.vector.complex.impl.*;
import org.apache.drill.exec.vector.complex.writer.*;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git
a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 537c7fd..cac87c5 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -296,7 +296,7 @@ package org.apache.drill.exec.vector;
}
public List<${friendlyType}> getObject(int index) {
- List<${friendlyType}> vals = Lists.newArrayList();
+ List<${friendlyType}> vals = new JsonStringArrayList();
int start = offsets.getAccessor().get(index) - sliceOffset;
int end = offsets.getAccessor().get(index+1) - sliceOffset;
for(int i = start; i < end; i++){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
index 4755e92..f2d66d2 100644
--- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
+++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
@@ -66,7 +66,12 @@ public class TypeHelper {
}
</#list>
</#list>
+ case MAP:
+ case REPEATMAP:
+ case LIST:
+ return new GenericAccessor(vector);
}
+
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
----------------------------------------------------------------------
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
new file mode 100644
index 0000000..bc378a1
--- /dev/null
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.apache.hadoop.io.Text;
+
+import java.util.ArrayList;
+
+public class JsonStringArrayList extends ArrayList {
+
+ private static ObjectMapper mapper;
+
+ static {
+ mapper = new ObjectMapper();
+ SimpleModule serializer = new SimpleModule("TextSerializer")
+ .addSerializer(Text.class, new TextSerializer());
+ mapper.registerModule(serializer);
+ }
+
+ @Override
+ public final String toString() {
+ try {
+ return mapper.writeValueAsString(this);
+ } catch(JsonProcessingException e) {
+ throw new IllegalStateException("Cannot serialize array list to JSON
string", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
----------------------------------------------------------------------
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
new file mode 100644
index 0000000..124252a
--- /dev/null
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.apache.hadoop.io.Text;
+
+import java.util.HashMap;
+
+/*
+ * Simple class that extends the regular java.util.HashMap but overrides the
+ * toString() method of the HashMap class to produce a JSON string instead
+ */
+public class JsonStringHashMap extends HashMap {
+
+ private static ObjectMapper mapper;
+
+ static {
+ mapper = new ObjectMapper();
+ SimpleModule serializer = new SimpleModule("TextSerializer")
+ .addSerializer(Text.class, new TextSerializer());
+ mapper.registerModule(serializer);
+ }
+
+ @Override
+ public final String toString() {
+ try {
+ return mapper.writeValueAsString(this);
+ } catch(JsonProcessingException e) {
+ throw new IllegalStateException("Cannot serialize hash map to JSON
string", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
----------------------------------------------------------------------
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
new file mode 100644
index 0000000..3070592
--- /dev/null
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+import org.apache.hadoop.io.Text;
+
+import java.io.IOException;
+
+public class TextSerializer extends StdSerializer<Text> {
+
+ public TextSerializer() {
+ super(Text.class);
+ }
+
+ @Override
+ public void serialize(Text text, JsonGenerator jsonGenerator,
SerializerProvider serializerProvider) throws IOException,
JsonGenerationException {
+ jsonGenerator.writeString(text.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 91c0be5..9b2e119 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -36,6 +36,7 @@ import
org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.util.JsonStringHashMap;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.RepeatedMapVector.MapSingleCopier;
import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl;
@@ -316,7 +317,7 @@ public class MapVector extends AbstractContainerVector {
@Override
public Object getObject(int index) {
- Map<String, Object> vv = Maps.newHashMap();
+ Map<String, Object> vv = new JsonStringHashMap();
for(Map.Entry<String, ValueVector> e : vectors.entrySet()){
ValueVector v = e.getValue();
String k = e.getKey();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
index 93930b5..88d858a 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -37,6 +37,7 @@ import
org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.apache.drill.exec.vector.ValueVector;
@@ -155,7 +156,7 @@ public class RepeatedListVector extends
AbstractContainerVector implements Repea
@Override
public Object getObject(int index) {
- List<Object> l = Lists.newArrayList();
+ List<Object> l = new JsonStringArrayList();
int end = offsets.getAccessor().get(index+1);
for(int i = offsets.getAccessor().get(index); i < end; i++){
l.add(vector.getAccessor().getObject(i));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 2492cc8..22471f0 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -36,6 +36,7 @@ import
org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.apache.drill.exec.vector.ValueVector;
@@ -348,7 +349,7 @@ public class RepeatedMapVector extends
AbstractContainerVector implements Repeat
@Override
public Object getObject(int index) {
- List<Object> l = Lists.newArrayList();
+ List<Object> l = new JsonStringArrayList();
int end = offsets.getAccessor().get(index+1);
for(int i = offsets.getAccessor().get(index); i < end; i++){
Map<String, Object> vv = Maps.newHashMap();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index 761bc79..e5d4b5b 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -221,7 +221,7 @@ public class JsonReader {
vh.start = 0;
vh.end = b.length;
list.varChar().write(vh);
-
+ break;
default:
throw new IllegalStateException("Unexpected token " +
parser.getCurrentToken());
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
----------------------------------------------------------------------
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
index 1372219..ad46326 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
@@ -27,6 +27,8 @@ public class JsonRecordReader2Test extends BaseTestQuery{
public void testComplexJsonInput() throws Exception{
// test("select z[0]['orange'] from cp.`jsoninput/input2.json` limit 10");
test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], z[0]['orange'],
z[1]['pink'] from cp.`jsoninput/input2.json` limit 10");
+ test("select x from cp.`jsoninput/input2.json`");
+
// test("select z[0] from cp.`jsoninput/input2.json` limit 10");
}
