DRILL-1060: Support ComplexToJson for Array Data Type
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5b57294b Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5b57294b Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5b57294b Branch: refs/heads/master Commit: 5b57294bfab1c9a6b82f7ad31189eef2b579abc5 Parents: 2dcf8cb Author: Aditya Kishore <[email protected]> Authored: Tue Jul 1 11:10:29 2014 -0700 Committer: Aditya Kishore <[email protected]> Committed: Wed Jul 9 11:55:46 2014 -0700 ---------------------------------------------------------------------- .../codegen/templates/HolderReaderImpl.java | 261 +++++++++++++++++++ .../codegen/templates/SingularReaderImpl.java | 210 --------------- .../src/main/codegen/templates/TypeHelper.java | 8 +- .../drill/exec/compile/JaninoClassCompiler.java | 2 +- .../apache/drill/exec/expr/ClassGenerator.java | 27 +- .../drill/exec/expr/EvaluationVisitor.java | 2 +- .../drill/exec/expr/fn/DrillFuncHolder.java | 13 +- .../impl/project/ProjectRecordBatch.java | 2 +- .../org/apache/drill/exec/util/VectorUtil.java | 2 +- .../complex/writer/TestComplexToJson.java | 77 ++++++ 10 files changed, 365 insertions(+), 239 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b57294b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java new file mode 100644 index 0000000..5f718a0 --- /dev/null +++ b/exec/java-exec/src/main/codegen/templates/HolderReaderImpl.java @@ -0,0 +1,261 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +<@pp.dropOutputFile /> +<#list vv.types as type> +<#list type.minor as minor> +<#list ["", "Nullable", "Repeated"] as holderMode> +<#assign nullMode = holderMode /> +<#if holderMode == "Repeated"><#assign nullMode = "Nullable" /></#if> + +<#assign lowerName = minor.class?uncap_first /> +<#if lowerName == "int" ><#assign lowerName = "integer" /></#if> +<#assign name = minor.class?cap_first /> +<#assign javaType = (minor.javaType!type.javaType) /> +<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> +<#assign safeType=friendlyType /> +<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if> + +<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${holderMode}${name}HolderReaderImpl.java" /> +<#include "/@includes/license.ftl" /> + +package org.apache.drill.exec.vector.complex.impl; + +<#include "/@includes/vv_imports.ftl" /> + +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.drill.exec.expr.holders.*; +import org.apache.hadoop.io.Text; +import org.joda.time.Period; +import org.mortbay.jetty.servlet.Holder; + +@SuppressWarnings("unused") +public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader { + + private ${nullMode}${name}Holder holder; +<#if holderMode == "Repeated" > + private int index = -1; + private ${holderMode}${name}Holder repeatedHolder; +</#if> + + public ${holderMode}${name}HolderReaderImpl(${holderMode}${name}Holder holder) { +<#if holderMode == "Repeated" > + this.holder = new ${nullMode}${name}Holder(); + this.repeatedHolder = holder; +<#else> + this.holder = holder; +</#if> + } + + @Override + public int size() { +<#if holderMode == "Repeated"> + return repeatedHolder.end - repeatedHolder.start; +<#else> + throw new UnsupportedOperationException("You can't call size on a Holder value reader."); +</#if> + } + + @Override + public boolean next() { +<#if holderMode == "Repeated"> + if(index + 1 < repeatedHolder.end) { + index++; + repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder); + return true; + } else { + return false; + } +<#else> + throw new UnsupportedOperationException("You can't call next on a single value reader."); +</#if> + + } + + @Override + public void setPosition(int index) { + throw new UnsupportedOperationException("You can't call next on a single value reader."); + } + + @Override + public MajorType getType() { +<#if holderMode == "Repeated"> + return this.repeatedHolder.getType(); +<#else> + return this.holder.getType(); +</#if> + } + + @Override + public boolean isSet() { + return this.holder.isSet(); + } + +<#if holderMode == "Repeated"> + @Override + public ${friendlyType} read${safeType}(int index){ + repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder); + ${friendlyType} value = read${safeType}(); + if (this.index > -1) { + repeatedHolder.vector.getAccessor().get(repeatedHolder.start + this.index, holder); + } + return value; + } +</#if> + + @Override + public ${friendlyType} read${safeType}(){ +<#if nullMode == "Nullable"> + if (!holder.isSet()) { + return null; + } +</#if> + +<#if type.major == "VarLen"> + + int length = holder.end - holder.start; + byte[] value = new byte [length]; + holder.buffer.getBytes(holder.start, value, 0, length); + +<#if minor.class == "VarBinary"> + return value; +<#elseif minor.class == "Var16Char"> + return new String(value); +<#elseif minor.class == "VarChar"> + Text text = new Text(); + text.set(value); + return text; +</#if> + +<#elseif minor.class == "Interval"> + Period p = new Period(); + return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliSeconds); + +<#elseif minor.class == "IntervalDay"> + Period p = new Period(); + return p.plusDays(holder.days).plusMillis(holder.milliSeconds); + +<#elseif minor.class == "Decimal9" || + minor.class == "Decimal18" > + BigInteger value = BigInteger.valueOf(holder.value); + return new BigDecimal(value, holder.scale); + +<#elseif minor.class == "Decimal28Dense" || + minor.class == "Decimal38Dense"> + return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromDense(holder.buffer, + holder.start, + holder.nDecimalDigits, + holder.scale, + holder.maxPrecision, + holder.WIDTH); + +<#elseif minor.class == "Decimal28Sparse" || + minor.class == "Decimal38Sparse"> + return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer, + holder.start, + holder.nDecimalDigits, + holder.scale); + +<#elseif minor.class == "Bit" > + return new Boolean(holder.value != 0); +<#else> + ${friendlyType} value = new ${friendlyType}(this.holder.value); + return value; +</#if> + + } + + @Override + public Object readObject() { +<#if holderMode == "Repeated" > + List<Object> valList = Lists.newArrayList(); + for (int i = repeatedHolder.start; i < repeatedHolder.end; i++) { + valList.add(repeatedHolder.vector.getAccessor().getObject(i)); + } + return valList; +<#else> + return readSingleObject(); +</#if> + } + + private Object readSingleObject() { +<#if nullMode == "Nullable"> + if (!holder.isSet()) { + return null; + } +</#if> + +<#if type.major == "VarLen"> + int length = holder.end - holder.start; + byte[] value = new byte [length]; + holder.buffer.getBytes(holder.start, value, 0, length); + +<#if minor.class == "VarBinary"> + return value; +<#elseif minor.class == "Var16Char"> + return new String(value); +<#elseif minor.class == "VarChar"> + Text text = new Text(); + text.set(value); + return text; +</#if> + +<#elseif minor.class == "Interval"> + Period p = new Period(); + return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliSeconds); + +<#elseif minor.class == "IntervalDay"> + Period p = new Period(); + return p.plusDays(holder.days).plusMillis(holder.milliSeconds); + +<#elseif minor.class == "Decimal9" || + minor.class == "Decimal18" > + BigInteger value = BigInteger.valueOf(holder.value); + return new BigDecimal(value, holder.scale); + +<#elseif minor.class == "Decimal28Dense" || + minor.class == "Decimal38Dense"> + return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromDense(holder.buffer, + holder.start, + holder.nDecimalDigits, + holder.scale, + holder.maxPrecision, + holder.WIDTH); + +<#elseif minor.class == "Decimal28Sparse" || + minor.class == "Decimal38Sparse"> + return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer, + holder.start, + holder.nDecimalDigits, + holder.scale); + +<#elseif minor.class == "Bit" > + return new Boolean(holder.value != 0); +<#else> + ${friendlyType} value = new ${friendlyType}(this.holder.value); + return value; +</#if> + } + +} + +</#list> +</#list> +</#list> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b57294b/exec/java-exec/src/main/codegen/templates/SingularReaderImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/SingularReaderImpl.java b/exec/java-exec/src/main/codegen/templates/SingularReaderImpl.java deleted file mode 100644 index 33cedba..0000000 --- a/exec/java-exec/src/main/codegen/templates/SingularReaderImpl.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -<@pp.dropOutputFile /> -<#list vv.types as type> -<#list type.minor as minor> -<#list ["", "Nullable"] as nullMode> -<#assign lowerName = minor.class?uncap_first /> -<#if lowerName == "int" ><#assign lowerName = "integer" /></#if> -<#assign name = minor.class?cap_first /> -<#assign javaType = (minor.javaType!type.javaType) /> -<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> -<#assign safeType=friendlyType /> -<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if> - -<@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/impl/${nullMode}${name}SingularReaderImpl.java" /> -<#include "/@includes/license.ftl" /> - -package org.apache.drill.exec.vector.complex.impl; - -<#include "/@includes/vv_imports.ftl" /> - -import java.math.BigDecimal; -import java.math.BigInteger; - -import org.apache.drill.exec.expr.holders.*; -import org.apache.hadoop.io.Text; -import org.joda.time.Period; -import org.mortbay.jetty.servlet.Holder; - -@SuppressWarnings("unused") -public class ${nullMode}${name}SingularReaderImpl extends AbstractFieldReader { - - private ${nullMode}${name}Holder holder; - - public ${nullMode}${name}SingularReaderImpl(${nullMode}${name}Holder holder) { - this.holder = holder; - } - - @Override - public int size() { - throw new UnsupportedOperationException("You can't call size on a singular value reader."); - } - - @Override - public boolean next() { - throw new UnsupportedOperationException("You can't call next on a single value reader."); - } - - @Override - public void setPosition(int index) { - throw new UnsupportedOperationException("You can't call next on a single value reader."); - } - - @Override - public MajorType getType() { - return this.holder.getType(); - } - - @Override - public boolean isSet() { - return this.holder.isSet(); - } - - @Override - public ${friendlyType} read${safeType}(){ - <#if nullMode == "Nullable"> - - if (!holder.isSet()) { - return null; - } - </#if> - - <#if type.major == "VarLen"> - - int length = holder.end - holder.start; - byte[] value = new byte [length]; - holder.buffer.getBytes(holder.start, value, 0, length); - - <#if minor.class == "VarBinary"> - return value; - <#elseif minor.class == "Var16Char"> - return new String(value); - <#elseif minor.class == "VarChar"> - Text text = new Text(); - text.set(value); - return text; - </#if> - - <#elseif minor.class == "Interval"> - Period p = new Period(); - return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliSeconds); - - <#elseif minor.class == "IntervalDay"> - Period p = new Period(); - return p.plusDays(holder.days).plusMillis(holder.milliSeconds); - - <#elseif minor.class == "Decimal9" || - minor.class == "Decimal18" > - BigInteger value = BigInteger.valueOf(holder.value); - return new BigDecimal(value, holder.scale); - - <#elseif minor.class == "Decimal28Dense" || - minor.class == "Decimal38Dense"> - return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromDense(holder.buffer, - holder.start, - holder.nDecimalDigits, - holder.scale, - holder.maxPrecision, - holder.WIDTH); - - <#elseif minor.class == "Decimal28Sparse" || - minor.class == "Decimal38Sparse"> - return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer, - holder.start, - holder.nDecimalDigits, - holder.scale); - - <#elseif minor.class == "Bit" > - return new Boolean(holder.value != 0); - <#else> - ${friendlyType} value = new ${friendlyType}(this.holder.value); - return value; - </#if> - - } - - @Override - public Object readObject(){ - - <#if nullMode == "Nullable"> - - if (!holder.isSet()) { - return null; - } - </#if> - - <#if type.major == "VarLen"> - - int length = holder.end - holder.start; - byte[] value = new byte [length]; - holder.buffer.getBytes(holder.start, value, 0, length); - - <#if minor.class == "VarBinary"> - return value; - <#elseif minor.class == "Var16Char"> - return new String(value); - <#elseif minor.class == "VarChar"> - Text text = new Text(); - text.set(value); - return text; - </#if> - - <#elseif minor.class == "Interval"> - Period p = new Period(); - return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliSeconds); - - <#elseif minor.class == "IntervalDay"> - Period p = new Period(); - return p.plusDays(holder.days).plusMillis(holder.milliSeconds); - - <#elseif minor.class == "Decimal9" || - minor.class == "Decimal18" > - BigInteger value = BigInteger.valueOf(holder.value); - return new BigDecimal(value, holder.scale); - - <#elseif minor.class == "Decimal28Dense" || - minor.class == "Decimal38Dense"> - return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromDense(holder.buffer, - holder.start, - holder.nDecimalDigits, - holder.scale, - holder.maxPrecision, - holder.WIDTH); - - <#elseif minor.class == "Decimal28Sparse" || - minor.class == "Decimal38Sparse"> - return org.apache.drill.common.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer, - holder.start, - holder.nDecimalDigits, - holder.scale); - - <#elseif minor.class == "Bit" > - return new Boolean(holder.value != 0); - <#else> - ${friendlyType} value = new ${friendlyType}(this.holder.value); - return value; - </#if> - } - -} - -</#list> -</#list> -</#list> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b57294b/exec/java-exec/src/main/codegen/templates/TypeHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java index 8c56d99..28cc428 100644 --- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java +++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java @@ -205,16 +205,18 @@ public class TypeHelper { throw new UnsupportedOperationException(); } - public static Class<?> getSingularReaderImpl( MinorType type, DataMode mode){ + public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){ switch (type) { <#list vv.types as type> <#list type.minor as minor> case ${minor.class?upper_case}: switch (mode) { case REQUIRED: - return ${minor.class}SingularReaderImpl.class; + return ${minor.class}HolderReaderImpl.class; case OPTIONAL: - return Nullable${minor.class}SingularReaderImpl.class; + return Nullable${minor.class}HolderReaderImpl.class; + case REPEATED: + return Repeated${minor.class}HolderReaderImpl.class; } </#list> </#list> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b57294b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java index 6622d77..e381e32 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/JaninoClassCompiler.java @@ -45,7 +45,7 @@ public class JaninoClassCompiler implements ClassCompiler{ public byte[][] getClassByteCode(final String className, final String code) throws CompileException, IOException, ClassNotFoundException, ClassTransformationException { if(logger.isDebugEnabled()){ - logger.debug("Compiling:\n {}", prefixLineNumbers(code)); + logger.debug("Compiling:\n{}", prefixLineNumbers(code)); } StringReader reader = new StringReader(code); Scanner scanner = new Scanner((String) null, reader); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b57294b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java index 7b1832c..0f5d1fd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java @@ -60,7 +60,6 @@ public class ClassGenerator<T>{ public static final GeneratorMapping DEFAULT_SCALAR_MAP = GM("doSetup", "doEval", null, null); public static final GeneratorMapping DEFAULT_CONSTANT_MAP = GM("doSetup", "doSetup", null, null); - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClassGenerator.class); public static enum BlockType {SETUP, EVAL, RESET, CLEANUP}; @@ -84,7 +83,6 @@ public class ClassGenerator<T>{ return new MappingSet("inIndex", "outIndex", DEFAULT_CONSTANT_MAP, DEFAULT_SCALAR_MAP); } - @SuppressWarnings("unchecked") ClassGenerator(CodeGenerator<T> codeGenerator, MappingSet mappingSet, SignatureHolder signature, EvaluationVisitor eval, JDefinedClass clazz, JCodeModel model) throws JClassAlreadyExistsException { this.codeGenerator = codeGenerator; @@ -155,16 +153,16 @@ public class ClassGenerator<T>{ String methodName = getCurrentMapping().getMethodName(BlockType.EVAL); this.blocks[sig.get(methodName)].addLast(block); } - + public void unNestEvalBlock() { String methodName = getCurrentMapping().getMethodName(BlockType.EVAL); this.blocks[sig.get(methodName)].removeLast(); } - + public JLabel getEvalBlockLabel (String prefix) { return getEvalBlock().label(prefix + labelIndex ++); } - + public JVar declareVectorValueSetupAndMember(String batchName, TypedFieldId fieldId){ return declareVectorValueSetupAndMember( DirectExpression.direct(batchName), fieldId); } @@ -205,7 +203,6 @@ public class ClassGenerator<T>{ getNextVar("tmp"), // invoke.invoke(vectorAccess)); - b._if(obj.eq(JExpr._null()))._then()._throw(JExpr._new(t).arg(JExpr.lit(String.format("Failure while loading vector %s with id: %s.", vv.name(), fieldId.toString())))); //b.assign(vv, JExpr.cast(retClass, ((JExpression) JExpr.cast(wrapperClass, obj) ).invoke(vectorAccess))); b.assign(vv, JExpr.cast(retClass, obj )); @@ -230,8 +227,6 @@ public class ClassGenerator<T>{ } } - - void flushCode(){ int i =0; for(CodeGeneratorMethod method : sig){ @@ -247,7 +242,6 @@ public class ClassGenerator<T>{ for(JBlock b : blocks[i++]){ if(!b.isEmpty()) m.body().add(b); } - } for(ClassGenerator<T> child : innerClasses.values()){ @@ -255,7 +249,6 @@ public class ClassGenerator<T>{ } } - public JCodeModel getModel() { return model; } @@ -345,10 +338,8 @@ public class ClassGenerator<T>{ return true; } - } - public static class HoldingContainer{ private final JVar holder; private final JFieldRef value; @@ -356,19 +347,24 @@ public class ClassGenerator<T>{ private final MajorType type; private boolean isConstant; private final boolean singularRepeated; + private final boolean isReader; public HoldingContainer(MajorType t, JVar holder, JFieldRef value, JFieldRef isSet) { - this(t, holder, value, isSet, false); + this(t, holder, value, isSet, false, false); } - public HoldingContainer(MajorType t, JVar holder, JFieldRef value, JFieldRef isSet, boolean singularRepeated) { - super(); + public HoldingContainer(MajorType t, JVar holder, JFieldRef value, JFieldRef isSet, boolean singularRepeated, boolean isReader) { this.holder = holder; this.value = value; this.isSet = isSet; this.type = t; this.isConstant = false; this.singularRepeated = singularRepeated; + this.isReader = isReader; + } + + public boolean isReader() { + return this.isReader; } public boolean isSingularRepeated(){ @@ -417,4 +413,5 @@ public class ClassGenerator<T>{ public JType getHolderType(MajorType t){ return TypeHelper.getHolderType(model, t.getMinorType(), t.getMode()); } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b57294b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java index 9cefaf3..5511cb2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java @@ -448,7 +448,7 @@ public class EvaluationVisitor { jc._then().assign(complexReader, expr); jc._else().assign(complexReader, nullReader); - HoldingContainer hc = new HoldingContainer(e.getMajorType(), complexReader, null, null, false); + HoldingContainer hc = new HoldingContainer(e.getMajorType(), complexReader, null, null, false, true); return hc; // //eval.assign(out.getHolder().ref("reader"), expr); } else { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b57294b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java index 2906705..f259d96 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java @@ -37,7 +37,6 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate; import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionCostCategory; import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; -import org.apache.drill.exec.vector.complex.impl.NullableBigIntSingularReaderImpl; import org.apache.drill.exec.vector.complex.reader.FieldReader; import com.google.common.base.Preconditions; @@ -147,9 +146,9 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder { ValueReference parameter = parameters[i]; HoldingContainer inputVariable = inputVariables[i]; - if (parameter.isFieldReader && ! Types.isComplex(inputVariable.getMajorType()) && ! Types.isRepeated(inputVariable.getMajorType())) { - JType singularReaderClass = g.getModel()._ref(TypeHelper.getSingularReaderImpl(inputVariable.getMajorType().getMinorType(), - inputVariable.getMajorType().getMode())); + if (parameter.isFieldReader && ! inputVariable.isReader() && ! Types.isComplex(inputVariable.getMajorType())) { + JType singularReaderClass = g.getModel()._ref(TypeHelper.getHolderReaderImpl(inputVariable.getMajorType().getMinorType(), + inputVariable.getMajorType().getMode())); JType fieldReadClass = g.getModel()._ref(FieldReader.class); sub.decl(fieldReadClass, parameter.name, JExpr._new(singularReaderClass).arg(inputVariable.getHolder())); } else { @@ -239,9 +238,9 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder { } public int getCostCategory() { - return this.costCategory.getValue(); + return this.costCategory.getValue(); } - + @Override public String toString() { final int maxLen = 10; @@ -291,7 +290,7 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder { return ref; } - + } public static class WorkspaceReference { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b57294b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index e6ddf90..985d96e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -307,7 +307,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{ List<NamedExpression> exprs = Lists.newArrayList(); for (MaterializedField field : incoming.getSchema()) { - if (Types.isComplex(field.getType())) { + if (Types.isComplex(field.getType()) || Types.isRepeated(field.getType())) { LogicalExpression convertToJson = FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON", field.getPath(), ExpressionPosition.UNKNOWN); String castFuncName = CastFunctions.getCastFunc(MinorType.VARCHAR); List<LogicalExpression> castArgs = Lists.newArrayList(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b57294b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java index 68f4550..98585c4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java @@ -87,7 +87,7 @@ public class VectorUtil { width += columnWidth + 2; formats.add("| %-" + columnWidth + "s"); MaterializedField field = vw.getValueVector().getField(); - columns.add(field.getPath().getAsUnescapedPath() + "<" + field.getType().getMinorType() + ">"); + columns.add(field.getPath().getAsUnescapedPath() + "<" + field.getType().getMinorType() + "(" + field.getType().getMode() + ")" + ">"); columnIndex++; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5b57294b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java new file mode 100644 index 0000000..0cffc88 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.complex.writer; + +import java.util.List; + +import org.apache.drill.BaseTestQuery; +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.exec.client.DrillClient; +import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef; +import org.apache.drill.exec.record.RecordBatchLoader; +import org.apache.drill.exec.rpc.user.QueryResultBatch; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestComplexToJson extends BaseTestQuery { + + @Test + public void test() throws Exception { + DrillClient parent_client = client; + + List<QueryResultBatch> results; + RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); + + client = new DrillClient(config, serviceSet.getCoordinator()); + client.setSupportComplexTypes(false); + client.connect(); + results = testSqlWithResults("select * from dfs.`[WORKING_PATH]/src/test/resources/store/text/data/regions.csv`"); + loader.load(results.get(0).getHeader().getDef(), results.get(0).getData()); + RecordBatchDef def = results.get(0).getHeader().getDef(); + // the entire row is returned as a single column + assertEquals(1, def.getFieldCount()); + // with setSupportComplexTypes == false, the column mode should be REQUIRED + assertTrue(def.getField(0).getMajorType().getMode() == DataMode.REQUIRED); + loader.clear(); + for(QueryResultBatch result : results) { + result.release(); + } + client.close(); + + client = new DrillClient(config, serviceSet.getCoordinator()); + client.setSupportComplexTypes(true); + client.connect(); + results = testSqlWithResults("select * from dfs.`[WORKING_PATH]/src/test/resources/store/text/data/regions.csv`"); + loader.load(results.get(0).getHeader().getDef(), results.get(0).getData()); + def = results.get(0).getHeader().getDef(); + // the entire row is returned as a single column + assertEquals(1, def.getFieldCount()); + // with setSupportComplexTypes == true, the column mode should be REPEATED + assertTrue(def.getField(0).getMajorType().getMode() == DataMode.REPEATED); + loader.clear(); + for(QueryResultBatch result : results) { + result.release(); + } + client.close(); + + client = parent_client; + } + +}
