DRILL-364: Fix VarChar casting
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/153c727a Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/153c727a Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/153c727a Branch: refs/heads/master Commit: 153c727a6d393203501f3cfed2535a0fb0601061 Parents: ee9eaa1 Author: Jinfeng Ni <[email protected]> Authored: Sun Feb 23 16:02:26 2014 -0800 Committer: Jacques Nadeau <[email protected]> Committed: Mon Mar 3 23:22:17 2014 -0800 ---------------------------------------------------------------------- .../common/expression/OutputTypeDeterminer.java | 20 +++++ .../common/expression/fn/CastFunctionDefs.java | 14 ++-- exec/java-exec/src/main/codegen/data/Casts.tdd | 4 +- .../exec/physical/impl/TestCastFunctions.java | 80 +++++++++++++++++++- .../functions/cast/testCastVarCharNull.json | 37 +++++++++ .../src/test/resources/jsoninput/input1.json | 6 ++ 6 files changed, 148 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java b/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java index 66523c4..69acf12 100644 --- a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java +++ b/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java @@ -74,4 +74,24 @@ public interface OutputTypeDeterminer { } } + public static class NullIfNullType implements OutputTypeDeterminer{ + public MinorType outputMinorType; + + + public NullIfNullType(MinorType outputType) { + super(); + this.outputMinorType = outputType; + } + + @Override + public MajorType getOutputType(List<LogicalExpression> expressions) { + for(LogicalExpression e : expressions){ + if(e.getMajorType().getMode() == DataMode.OPTIONAL){ + return Types.optional(outputMinorType); + } + } + return Types.required(outputMinorType); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java index 6a98f94..4be3820 100644 --- a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java +++ b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java @@ -25,13 +25,13 @@ import org.apache.drill.common.types.TypeProtos.MinorType; public class CastFunctionDefs implements CallProvider{ - private static final FunctionDefinition CAST_BIG_INT = FunctionDefinition.simple("castBIGINT", new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_BIGINT); - private static final FunctionDefinition CAST_INT = FunctionDefinition.simple("castINT", new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_INT); - private static final FunctionDefinition CAST_FLOAT4 = FunctionDefinition.simple("castFLOAT4", new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_FLOAT4); - private static final FunctionDefinition CAST_FLOAT8 = FunctionDefinition.simple("castFLOAT8", new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_FLOAT8); - private static final FunctionDefinition CAST_VARCHAR = FunctionDefinition.simple("castVARCHAR", new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VARCHAR); - private static final FunctionDefinition CAST_VARBINARY = FunctionDefinition.simple("castVARBINARY", new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VARBINARY); - private static final FunctionDefinition CAST_VAR16CHAR = FunctionDefinition.simple("castVAR16CHAR", new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VAR16CHAR); + private static final FunctionDefinition CAST_BIG_INT = FunctionDefinition.simple("castBIGINT", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.BIGINT)); + private static final FunctionDefinition CAST_INT = FunctionDefinition.simple("castINT", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.INT)); + private static final FunctionDefinition CAST_FLOAT4 = FunctionDefinition.simple("castFLOAT4", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.FLOAT4)); + private static final FunctionDefinition CAST_FLOAT8 = FunctionDefinition.simple("castFLOAT8", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.FLOAT8)); + private static final FunctionDefinition CAST_VARCHAR = FunctionDefinition.simple("castVARCHAR", new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR)); + private static final FunctionDefinition CAST_VARBINARY = FunctionDefinition.simple("castVARBINARY", new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VARBINARY)); + private static final FunctionDefinition CAST_VAR16CHAR = FunctionDefinition.simple("castVAR16CHAR", new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VAR16CHAR)); @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/main/codegen/data/Casts.tdd ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/data/Casts.tdd b/exec/java-exec/src/main/codegen/data/Casts.tdd index ceb9cde..fde63cf 100644 --- a/exec/java-exec/src/main/codegen/data/Casts.tdd +++ b/exec/java-exec/src/main/codegen/data/Casts.tdd @@ -39,11 +39,11 @@ {from: "VarBinary", to: "Float8", major: "SrcVarlen", javaType:"Double", parse:"Double"}, {from: "BigInt", to: "VarChar", major: "TargetVarlen", javaType: "Long", bufferLength:"20"}, - {from: "Int", to: "VarChar", major: "TargetVarlen", javaType: "Integer", bufferLength:"10"}, + {from: "Int", to: "VarChar", major: "TargetVarlen", javaType: "Integer", bufferLength:"11"}, {from: "Float4", to: "VarChar", major: "TargetVarlen", javaType: "Float", bufferLength:"100"}, {from: "Float8", to: "VarChar", major: "TargetVarlen", javaType: "Double", bufferLength:"100"}, {from: "BigInt", to: "VarBinary", major: "TargetVarlen", javaType: "Long", bufferLength:"20"}, - {from: "Int", to: "VarBinary", major: "TargetVarlen", javaType: "Integer", bufferLength:"10"}, + {from: "Int", to: "VarBinary", major: "TargetVarlen", javaType: "Integer", bufferLength:"11"}, {from: "Float4", to: "VarBinary", major: "TargetVarlen", javaType: "Float", bufferLength:"100"}, {from: "Float8", to: "VarBinary", major: "TargetVarlen", javaType: "Double", bufferLength:"100"}, http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java index 3010c37..4d18815 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java @@ -19,6 +19,9 @@ package org.apache.drill.exec.physical.impl; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; + +import java.util.List; + import mockit.Injectable; import mockit.NonStrictExpectations; @@ -26,6 +29,7 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.util.FileUtils; +import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.expr.holders.BigIntHolder; import org.apache.drill.exec.expr.holders.Float4Holder; @@ -38,34 +42,44 @@ import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.physical.base.FragmentRoot; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.proto.BitControl.PlanFragment; import org.apache.drill.exec.proto.CoordinationProtos; -import org.apache.drill.exec.proto.ExecProtos.FragmentHandle; +import org.apache.drill.exec.proto.UserProtos; +import org.apache.drill.exec.record.RecordBatchLoader; +import org.apache.drill.exec.record.VectorAccessible; +import org.apache.drill.exec.record.VectorWrapper; +import org.apache.drill.exec.rpc.user.QueryResultBatch; import org.apache.drill.exec.rpc.user.UserServer; +import org.apache.drill.exec.server.Drillbit; import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.server.RemoteServiceSet; import org.apache.drill.exec.vector.BigIntVector; import org.apache.drill.exec.vector.Float4Vector; import org.apache.drill.exec.vector.Float8Vector; import org.apache.drill.exec.vector.IntVector; +import org.apache.drill.exec.vector.NullableVarCharVector; +import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VarBinaryVector; import org.apache.drill.exec.vector.VarCharVector; import org.junit.After; +import org.junit.Ignore; import org.junit.Test; import com.codahale.metrics.MetricRegistry; import com.google.common.base.Charsets; import com.google.common.io.Files; -public class TestCastFunctions { +public class TestCastFunctions extends PopUnitTestBase{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class); DrillConfig c = DrillConfig.create(); + @Test // cast to bigint. public void testCastBigInt(@Injectable final DrillbitContext bitContext, @Injectable UserServer.UserClientConnection connection) throws Throwable{ - new NonStrictExpectations(){{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); @@ -342,7 +356,6 @@ public class TestCastFunctions { } @Test(expected = NumberFormatException.class) - //nested: cast is nested in another cast, or another function. public void testCastNumException(@Injectable final DrillbitContext bitContext, @Injectable UserServer.UserClientConnection connection) throws Throwable{ @@ -370,6 +383,65 @@ public class TestCastFunctions { } + @Test + public void testCastFromNullablCol() throws Throwable { + RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); + + try(Drillbit bit = new Drillbit(CONFIG, serviceSet); + DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) { + bit.run(); + + client.connect(); + List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, + Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarCharNull.json"), Charsets.UTF_8).replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/jsoninput/input1.json").toURI().toString())); + + QueryResultBatch batch = results.get(0); + + RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator()); + batchLoader.load(batch.getHeader().getDef(), batch.getData()); + + Object [][] result = getRunResult(batchLoader); + + Object [][] expected = new Object[2][2]; + + expected[0][0] = new String("2001"); + expected[0][1] = new String("1.2"); + + expected[1][0] = new String("-2002"); + expected[1][1] = new String("-1.2"); + + assertEquals(result.length, expected.length); + assertEquals(result[0].length, expected[0].length); + + for (int i = 0; i<result.length; i++ ) { + for (int j = 0; j<result[0].length; j++) { + assertEquals(String.format("Column %s at row %s have wrong result", j, i), result[i][j], expected[i][j]); + } + } + } + } + + private Object[][] getRunResult(VectorAccessible va) { + int size = 0; + for (VectorWrapper v : va) { + size++; + } + + Object[][] res = new Object [va.getRecordCount()][size]; + for (int j = 0; j < va.getRecordCount(); j++) { + int i = 0; + for (VectorWrapper v : va) { + Object o = v.getValueVector().getAccessor().getObject(j); + if (o instanceof byte[]) { + res[j][i++] = new String((byte[]) o); + } else { + res[j][i++] = o; + } + } + } + return res; + } + @After public void tearDown() throws Exception{ // pause to get logger to catch up. http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json b/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json new file mode 100644 index 0000000..030b696 --- /dev/null +++ b/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json @@ -0,0 +1,37 @@ +{ + head:{ + type:"APACHE_DRILL_PHYSICAL", + version:"1", + generator:{ + type:"manual" + } + }, + graph:[ + { + @id:1, + pop:"json-scan", + entries: [ + { + path : "#{TEST_FILE}" + } + ], + storageengine: { + "type": "json", + "dfsName": "file:///" + } + }, { + @id:2, + child: 1, + pop:"project", + exprs: [ + { ref: "int2varchar", expr:"cast(integer as varchar(20))" }, + { ref: "float2varchar", expr:"cast(float as varchar(20))" } + ] + }, + { + @id: 3, + child: 2, + pop: "screen" + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/resources/jsoninput/input1.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/jsoninput/input1.json b/exec/java-exec/src/test/resources/jsoninput/input1.json new file mode 100644 index 0000000..e9bde7e --- /dev/null +++ b/exec/java-exec/src/test/resources/jsoninput/input1.json @@ -0,0 +1,6 @@ +{ "integer" : 2001, + "float" : 1.2 +} +{ "integer" : -2002, + "float" : -1.2 +}
