Repository: incubator-drill Updated Branches: refs/heads/master 7ad6de699 -> 241340728
Join rel > logical conversion fixes Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a4199dde Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a4199dde Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a4199dde Branch: refs/heads/master Commit: a4199dde428be0dfb7da990332e693e59a9cd13e Parents: 7ad6de6 Author: Jacques Nadeau <[email protected]> Authored: Mon Mar 17 21:29:42 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Mon Mar 17 21:29:42 2014 -0700 ---------------------------------------------------------------------- .../impl/project/ProjectRecordBatch.java | 10 +- .../exec/planner/logical/DrillImplementor.java | 3 +- .../exec/planner/logical/DrillJoinRel.java | 2 +- .../drill/exec/planner/logical/DrillTable.java | 37 +++++++ .../exec/planner/logical/DynamicDrillTable.java | 4 +- .../planner/logical/RelDataTypeDrillImpl.java | 92 +++------------- .../exec/planner/logical/RelDataTypeHolder.java | 85 +++++++++++++++ .../org/apache/drill/TestExampleQueries.java | 24 ++++- .../drill/exec/record/vector/TestLoad.java | 105 +++++++++---------- 9 files changed, 222 insertions(+), 140 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a4199dde/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index 509d13b..4384f0b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -88,6 +88,14 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{ } return ref; } + + private boolean isAnyWildcard(List<NamedExpression> exprs){ + for(NamedExpression e : exprs){ + if(isWildcard(e)) return true; + } + return false; + } + private boolean isWildcard(NamedExpression ex){ LogicalExpression expr = ex.getExpr(); LogicalExpression ref = ex.getRef(); @@ -111,7 +119,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{ final ClassGenerator<Projector> cg = CodeGenerator.getRoot(Projector.TEMPLATE_DEFINITION, context.getFunctionRegistry()); - if(exprs.size() == 1 && isWildcard(exprs.get(0))){ + if(isAnyWildcard(exprs)){ for(VectorWrapper<?> wrapper : incoming){ ValueVector vvIn = wrapper.getValueVector(); TransferPair tp = wrapper.getValueVector().getTransferPair(new FieldReference(vvIn.getField().getName())); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a4199dde/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillImplementor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillImplementor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillImplementor.java index acd218c..6a7c8e4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillImplementor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillImplementor.java @@ -37,6 +37,7 @@ public class DrillImplementor { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillImplementor.class); private Set<DrillTable> tables = Sets.newHashSet(); + private Set<String> storageEngineNames = Sets.newHashSet(); private LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(); private LogicalPlan plan; private final DrillParseContext context; @@ -52,7 +53,7 @@ public class DrillImplementor { } public void registerSource(DrillTable table){ - if(tables.add(table)){ + if(tables.add(table) && storageEngineNames.add(table.getStorageEngineName())){ planBuilder.addStorageEngine(table.getStorageEngineName(), table.getStorageEngineConfig()); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a4199dde/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java index c08712e..16e9d47 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java @@ -119,7 +119,7 @@ public class DrillJoinRel extends JoinRelBase implements DrillRel { Project.Builder builder = Project.builder(); builder.setInput(inputOp); for (Pair<String, String> pair : Pair.zip(inputFields, outputFields)) { - builder.addExpr(new FieldReference(pair.right), new FieldReference(pair.left)); + builder.addExpr(new FieldReference("output." + pair.right), new FieldReference(pair.left)); } return builder.build(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a4199dde/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java index 02ca0ad..bad6f74 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java @@ -69,6 +69,43 @@ public abstract class DrillTable implements Table{ return TableType.TABLE; } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((selection == null) ? 0 : selection.hashCode()); + result = prime * result + ((storageEngineConfig == null) ? 0 : storageEngineConfig.hashCode()); + result = prime * result + ((storageEngineName == null) ? 0 : storageEngineName.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + DrillTable other = (DrillTable) obj; + if (selection == null) { + if (other.selection != null) + return false; + } else if (!selection.equals(other.selection)) + return false; + if (storageEngineConfig == null) { + if (other.storageEngineConfig != null) + return false; + } else if (!storageEngineConfig.equals(other.storageEngineConfig)) + return false; + if (storageEngineName == null) { + if (other.storageEngineName != null) + return false; + } else if (!storageEngineName.equals(other.storageEngineName)) + return false; + return true; + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a4199dde/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java index c50e67c..36b554f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java @@ -24,12 +24,14 @@ import org.eigenbase.reltype.RelDataTypeFactory; public class DynamicDrillTable extends DrillTable{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DynamicDrillTable.class); + private RelDataTypeHolder holder = new RelDataTypeHolder(); + public DynamicDrillTable(String storageEngineName, Object selection, StoragePluginConfig storageEngineConfig) { super(storageEngineName, selection, storageEngineConfig); } @Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { - return new RelDataTypeDrillImpl(typeFactory); + return new RelDataTypeDrillImpl(holder, typeFactory); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a4199dde/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeDrillImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeDrillImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeDrillImpl.java index 835e4a5..bc0f34e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeDrillImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeDrillImpl.java @@ -17,20 +17,13 @@ */ package org.apache.drill.exec.planner.logical; +import java.util.List; - - -import org.eigenbase.reltype.*; -import org.eigenbase.sql.SqlCollation; -import org.eigenbase.sql.SqlIdentifier; -import org.eigenbase.sql.SqlIntervalQualifier; +import org.eigenbase.reltype.RelDataTypeFactory; +import org.eigenbase.reltype.RelDataTypeField; +import org.eigenbase.reltype.RelDataTypeImpl; import org.eigenbase.sql.type.SqlTypeName; -import java.nio.charset.Charset; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - /* We use an instance of this class as the row type for * Drill table. Since we don't know the schema before hand * whenever optiq requires us to validate that a field exists @@ -39,90 +32,33 @@ import java.util.List; */ public class RelDataTypeDrillImpl extends RelDataTypeImpl { - private RelDataTypeField defaultField; - RelDataTypeFactory typeFactory; - List<RelDataTypeField> drillFieldList = new LinkedList<>(); - List<String> drillfieldNames = new LinkedList<>(); - - - public RelDataTypeDrillImpl(RelDataTypeFactory typeFactory) { + private final RelDataTypeFactory typeFactory; + private final RelDataTypeHolder holder; + + public RelDataTypeDrillImpl(RelDataTypeHolder holder, RelDataTypeFactory typeFactory) { this.typeFactory = typeFactory; + this.holder = holder; computeDigest(); } - + @Override public List<RelDataTypeField> getFieldList() { - - if (drillFieldList.size() == 0) - { - /* By default we only have a single row in drill of 'ANY' type - * (mainly for select * type queries) - */ - defaultField = new RelDataTypeFieldImpl("*", 0, typeFactory.createSqlType(SqlTypeName.ANY)); - - drillFieldList.add(defaultField); - drillfieldNames.add("*"); - } - return drillFieldList; + return holder.getFieldList(typeFactory); } - @Override public int getFieldCount() { - return drillFieldList.size(); + return holder.getFieldCount(); } -// @Override -// public int getFieldOrdinal(String fieldName, boolean caseSensitive) { -// -// /* Get the list of fields and return the -// * index if the field exists -// */ -// for (RelDataTypeField field : drillFieldList) { -// if (field.getName().equals(fieldName)) -// return field.getIndex(); -// } -// -// /* Couldn't find the field in our list, return -1 -// * Unsure if I should add it to our list of fields -// */ -// return -1; -// } - @Override - /** - * - */ public RelDataTypeField getField(String fieldName, boolean caseSensitive) { - - /* First check if this field name exists in our field list */ - for (RelDataTypeField field : drillFieldList) - { - if (field.getName().equals(fieldName)) - return field; - } - - /* This field does not exist in our field list add it */ - RelDataTypeField newField = new RelDataTypeFieldImpl(fieldName, drillFieldList.size(), typeFactory.createSqlType(SqlTypeName.ANY)); - - /* Add it to the list of fields */ - drillFieldList.add(newField); - - /* Add the name to our list of field names */ - drillfieldNames.add(fieldName); - - return newField; + return holder.getField(typeFactory, fieldName); } - @Override public List<String> getFieldNames() { - - if (drillfieldNames.size() == 0) { - drillfieldNames.add("*"); - } - - return drillfieldNames; + return holder.getFieldNames(); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a4199dde/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeHolder.java new file mode 100644 index 0000000..038f3de --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/RelDataTypeHolder.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.logical; + +import java.util.List; + +import org.eigenbase.reltype.RelDataTypeFactory; +import org.eigenbase.reltype.RelDataTypeField; +import org.eigenbase.reltype.RelDataTypeFieldImpl; +import org.eigenbase.sql.type.SqlTypeName; + +import com.google.common.collect.Lists; + +public class RelDataTypeHolder { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RelDataTypeHolder.class); + + List<String> fieldNames = Lists.newArrayList(); + + public List<RelDataTypeField> getFieldList(RelDataTypeFactory typeFactory) { + + addStarIfEmpty(); + + List<RelDataTypeField> fields = Lists.newArrayList(); + + int i = 0; + for (String fieldName : fieldNames) { + + RelDataTypeField field = new RelDataTypeFieldImpl(fieldName, i, typeFactory.createSqlType(SqlTypeName.ANY)); + fields.add(field); + i++; + } + + return fields; + } + + public int getFieldCount() { + addStarIfEmpty(); + return fieldNames.size(); + } + + private void addStarIfEmpty(){ + if (fieldNames.isEmpty()) fieldNames.add("*"); + } + + public RelDataTypeField getField(RelDataTypeFactory typeFactory, String fieldName) { + + /* First check if this field name exists in our field list */ + int i = 0; + for (String name : fieldNames) { + if (name.equalsIgnoreCase(fieldName)) { + return new RelDataTypeFieldImpl(name, i, typeFactory.createSqlType(SqlTypeName.ANY)); + } + i++; + } + + /* This field does not exist in our field list add it */ + RelDataTypeField newField = new RelDataTypeFieldImpl(fieldName, fieldNames.size(), + typeFactory.createSqlType(SqlTypeName.ANY)); + + /* Add the name to our list of field names */ + fieldNames.add(fieldName); + + return newField; + } + + public List<String> getFieldNames() { + addStarIfEmpty(); + return fieldNames; + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a4199dde/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java index c7fde56..e8b9687 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java @@ -34,6 +34,19 @@ public class TestExampleQueries { } @Test + public void testJoin() throws Exception{ + test("SELECT\n" + + " nations.N_NAME,\n" + + " regions.R_NAME\n" + + "FROM\n" + + " dfs.`/opt/drill/sample-data/nation.parquet` nations\n" + + "JOIN\n" + + " dfs.`/opt/drill/sample-data/region.parquet` regions\n" + + " on nations.N_REGIONKEY = regions.R_REGIONKEY"); + } + + + @Test public void testWhere() throws Exception{ test("select * from cp.`employee.json` where employee_id > 10 and employee_id < 20"); } @@ -44,8 +57,15 @@ public class TestExampleQueries { } private void test(String sql) throws Exception{ - QuerySubmitter s = new QuerySubmitter(); - s.submitQuery(null, sql, "sql", null, true, 1, "tsv"); + boolean good = false; + + try{ + QuerySubmitter s = new QuerySubmitter(); + s.submitQuery(null, sql, "sql", null, true, 1, "tsv"); + good = true; + }finally{ + if(!good) Thread.sleep(2000); + } } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a4199dde/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java index 3d072cd..7dddfa0 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java @@ -17,15 +17,15 @@ */ package org.apache.drill.exec.record.vector; -import com.google.common.collect.Lists; - +import static org.junit.Assert.assertEquals; import io.netty.buffer.ByteBuf; -import org.apache.drill.common.config.DrillConfig; +import java.util.List; + import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.common.types.TypeProtos.*; +import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.TopLevelAllocator; @@ -33,29 +33,25 @@ import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.record.WritableBatch; -import org.apache.drill.exec.vector.*; +import org.apache.drill.exec.vector.AllocationHelper; +import org.apache.drill.exec.vector.IntVector; +import org.apache.drill.exec.vector.NullableVarCharVector; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.VarCharVector; import org.junit.Test; -import java.util.List; - -import static org.junit.Assert.assertEquals; - +import com.google.common.collect.Lists; public class TestLoad { @Test - public void testLoadValueVector() { + public void testLoadValueVector() throws Exception { BufferAllocator allocator = new TopLevelAllocator(); - ValueVector fixedV = new IntVector( - MaterializedField.create(new SchemaPath("ints", ExpressionPosition.UNKNOWN), Types.required(MinorType.INT)), - allocator); - ValueVector varlenV = new VarCharVector( - MaterializedField.create(new SchemaPath("chars", ExpressionPosition.UNKNOWN), Types.required(MinorType.VARCHAR)), - allocator - ); - ValueVector nullableVarlenV = new NullableVarCharVector( - MaterializedField.create(new SchemaPath("chars", ExpressionPosition.UNKNOWN), Types.optional(MinorType.VARCHAR)), - allocator - ); + ValueVector fixedV = new IntVector(MaterializedField.create(new SchemaPath("ints", ExpressionPosition.UNKNOWN), + Types.required(MinorType.INT)), allocator); + ValueVector varlenV = new VarCharVector(MaterializedField.create( + new SchemaPath("chars", ExpressionPosition.UNKNOWN), Types.required(MinorType.VARCHAR)), allocator); + ValueVector nullableVarlenV = new NullableVarCharVector(MaterializedField.create(new SchemaPath("chars", + ExpressionPosition.UNKNOWN), Types.optional(MinorType.VARCHAR)), allocator); List<ValueVector> vectors = Lists.newArrayList(fixedV, varlenV, nullableVarlenV); for (ValueVector v : vectors) { @@ -78,49 +74,46 @@ public class TestLoad { index += byteBufs[i].writerIndex(); } byteBuf.writerIndex(bytes); - try { - batchLoader.load(writableBatch.getDef(), byteBuf); - boolean firstColumn = true; - int recordCount = 0; + + batchLoader.load(writableBatch.getDef(), byteBuf); + boolean firstColumn = true; + int recordCount = 0; + for (VectorWrapper<?> v : batchLoader) { + if (firstColumn) { + firstColumn = false; + } else { + System.out.print("\t"); + } + System.out.print(v.getField().getName()); + System.out.print("["); + System.out.print(v.getField().getType().getMinorType()); + System.out.print("]"); + } + + System.out.println(); + for (int r = 0; r < batchLoader.getRecordCount(); r++) { + boolean first = true; + recordCount++; for (VectorWrapper<?> v : batchLoader) { - if (firstColumn) { - firstColumn = false; + if (first) { + first = false; } else { System.out.print("\t"); } - System.out.print(v.getField().getName()); - System.out.print("["); - System.out.print(v.getField().getType().getMinorType()); - System.out.print("]"); - } - - System.out.println(); - for (int r = 0; r < batchLoader.getRecordCount(); r++) { - boolean first = true; - recordCount++; - for (VectorWrapper<?> v : batchLoader) { - if (first) { - first = false; - } else { - System.out.print("\t"); - } - ValueVector.Accessor accessor = v.getValueVector().getAccessor(); - if (v.getField().getType().getMinorType() == TypeProtos.MinorType.VARCHAR) { - Object obj = accessor.getObject(r) ; - if(obj != null) - System.out.print(new String((byte[]) accessor.getObject(r))); - else - System.out.print("NULL"); - } else { + ValueVector.Accessor accessor = v.getValueVector().getAccessor(); + if (v.getField().getType().getMinorType() == TypeProtos.MinorType.VARCHAR) { + Object obj = accessor.getObject(r); + if (obj != null) System.out.print(accessor.getObject(r)); - } + else + System.out.print("NULL"); + } else { + System.out.print(accessor.getObject(r)); } - if (!first) System.out.println(); } - assertEquals(100, recordCount); - } catch (Exception e) { - e.printStackTrace(); + if (!first) System.out.println(); } + assertEquals(100, recordCount); } }
