http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java index 20609b8..3981ee1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaDrillTable.java @@ -27,8 +27,8 @@ public class InfoSchemaDrillTable extends DrillTable{ private final SelectedTable table; - public InfoSchemaDrillTable(String storageEngineName, SelectedTable selection, StoragePluginConfig storageEngineConfig) { - super(storageEngineName, selection, storageEngineConfig); + public InfoSchemaDrillTable(InfoSchemaStoragePlugin plugin, String storageEngineName, SelectedTable selection, StoragePluginConfig storageEngineConfig) { + super(storageEngineName, plugin, selection); this.table = selection; }
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java index 891419d..a7d7c5c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java @@ -24,6 +24,8 @@ import java.util.Set; import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaPlus; +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.common.logical.data.Scan; import org.apache.drill.exec.planner.logical.DrillTable; import org.apache.drill.exec.server.DrillbitContext; @@ -53,25 +55,30 @@ public class InfoSchemaStoragePlugin extends AbstractStoragePlugin{ } @Override - public InfoSchemaGroupScan getPhysicalScan(Scan scan) throws IOException { - SelectedTable table = scan.getSelection().getWith(context.getConfig(), SelectedTable.class); + public InfoSchemaGroupScan getPhysicalScan(JSONOptions selection) throws IOException { + SelectedTable table = selection.getWith(context.getConfig(), SelectedTable.class); return new InfoSchemaGroupScan(table); } @Override + public StoragePluginConfig getConfig() { + return this.config; + } + + @Override public Schema createAndAddSchema(SchemaPlus parent) { - Schema s = new ISchema(parent); + Schema s = new ISchema(parent, this); parent.add(s); return s; } private class ISchema extends AbstractSchema{ private Map<String, InfoSchemaDrillTable> tables; - public ISchema(SchemaPlus parent){ + public ISchema(SchemaPlus parent, InfoSchemaStoragePlugin plugin){ super(new SchemaHolder(parent), "INFORMATION_SCHEMA"); Map<String, InfoSchemaDrillTable> tbls = Maps.newHashMap(); for(SelectedTable tbl : SelectedTable.values()){ - tbls.put(tbl.name(), new InfoSchemaDrillTable("INFORMATION_SCHEMA", tbl, config)); + tbls.put(tbl.name(), new InfoSchemaDrillTable(plugin, "INFORMATION_SCHEMA", tbl, config)); } this.tables = ImmutableMap.copyOf(tbls); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java index 47218fe..c20c134 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java @@ -96,10 +96,15 @@ public class ParquetGroupScan extends AbstractGroupScan { } @JsonProperty("format") - public ParquetFormatConfig getEngineConfig() { + public ParquetFormatConfig getFormatConfig() { return this.formatConfig; } + @JsonProperty("storage") + public StoragePluginConfig getEngineConfig() { + return this.formatPlugin.getStorageConfig(); + } + @JsonCreator public ParquetGroupScan( // @JsonProperty("entries") List<ReadEntryWithPath> entries, // http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index 90b2a4d..5d53b7d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -346,6 +346,16 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{ private void runSQL(String sql) { try{ + DrillSqlWorker sqlWorker = new DrillSqlWorker(context.getFactory(), context.getFunctionRegistry()); + + PhysicalPlan physical = sqlWorker.getPhysicalPlan(sql, context); + + if(logger.isDebugEnabled()) { + logger.debug("Distributed Physical {}", context.getConfig().getMapper().writeValueAsString(physical)); + System.out.println(context.getConfig().getMapper().writeValueAsString(physical)); + } + + runPhysicalPlan(physical); }catch(Exception e){ fail("Failure while parsing sql.", e); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java index eaf921d..b305d0d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java @@ -56,7 +56,7 @@ import com.google.common.collect.Maps; /** * Each Foreman holds its own fragment manager. This manages the events associated with execution of a particular query across all fragments. */ -class QueryManager implements FragmentStatusListener{ +public class QueryManager implements FragmentStatusListener{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryManager.class); public Map<FragmentHandle, FragmentData> map = Maps.newHashMap(); // doesn't need to be thread safe as map is generated in a single thread and then accessed by multiple threads for reads only. http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java index 7199c65..5baaf63 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java @@ -19,23 +19,21 @@ package org.apache.drill; import org.apache.drill.common.util.TestTools; import org.apache.drill.exec.client.QuerySubmitter; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; -public class TestExampleQueries extends BaseTestQuery{ +public class TestExampleQueries { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class); - @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(20000); + @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(10000000); @Test public void testSelectWithLimit() throws Exception{ test("select * from cp.`employee.json` limit 5"); } - - - @Test public void testJoin() throws Exception{ test("SELECT\n" + @@ -59,4 +57,17 @@ public class TestExampleQueries extends BaseTestQuery{ test("select marital_status, COUNT(1) as cnt from cp.`employee.json` group by marital_status"); } + private void test(String sql) throws Exception{ + boolean good = false; + sql = sql.replace("[WORKING_PATH]", TestTools.getWorkingPath()); + + try{ + QuerySubmitter s = new QuerySubmitter(); + s.submitQuery(null, sql, "sql", null, true, 1, "tsv"); + good = true; + }finally{ + if(!good) Thread.sleep(2000); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/test/java/org/apache/drill/TestTpchQueries.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchQueries.java index 346ed56..8643fa3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchQueries.java @@ -24,6 +24,7 @@ public class TestTpchQueries extends BaseTestQuery{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchQueries.class); @Test + @Ignore public void tpch01() throws Exception{ testSqlFromFile("queries/tpch/01.sql"); } @@ -53,6 +54,7 @@ public class TestTpchQueries extends BaseTestQuery{ } @Test // DRILL-356 + @Ignore public void tpch06() throws Exception{ testSqlFromFile("queries/tpch/06.sql"); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java index 6e681e1..8e92181 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java @@ -28,6 +28,7 @@ import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.util.FileUtils; +import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.TopLevelAllocator; @@ -38,12 +39,17 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.ImplCreator; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.proto.ExecProtos; +import org.apache.drill.exec.proto.UserProtos; import org.apache.drill.exec.proto.BitControl.PlanFragment; +import org.apache.drill.exec.rpc.user.QueryResultBatch; import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; +import org.apache.drill.exec.server.Drillbit; import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.server.RemoteServiceSet; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.vector.ValueVector; import org.junit.AfterClass; @@ -55,7 +61,7 @@ import com.google.common.io.Files; import com.codahale.metrics.MetricRegistry; -public class TestMergeJoin { +public class TestMergeJoin extends PopUnitTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestMergeJoin.class); DrillConfig c = DrillConfig.create(); @@ -292,12 +298,35 @@ public class TestMergeJoin { } + @Test + public void testMergeJoinEmptyBatch() throws Exception { + RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); + + try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet); + DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) { + + bit1.run(); + client.connect(); + List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, + Files.toString(FileUtils.getResourceAsFile("/join/merge_join_empty_batch.json"), + Charsets.UTF_8)); + int count = 0; + for(QueryResultBatch b : results) { + if (b.getHeader().getRowCount() != 0) + count += b.getHeader().getRowCount(); + } + assertEquals(0, count); + } + } + + @AfterClass public static void tearDown() throws Exception{ // pause to get logger to catch up. Thread.sleep(1000); } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/test/java/org/apache/drill/exec/planner/physical/TestPhysicalPlanning.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/physical/TestPhysicalPlanning.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/physical/TestPhysicalPlanning.java deleted file mode 100644 index 68d1eef..0000000 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/physical/TestPhysicalPlanning.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.apache.drill.exec.planner.physical; - -import mockit.NonStrictExpectations; - -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.expression.FunctionRegistry; -import org.apache.drill.exec.memory.TopLevelAllocator; -import org.apache.drill.exec.planner.sql.DrillSqlWorker; -import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.StoragePluginRegistry; -import org.junit.Test; - -import com.codahale.metrics.MetricRegistry; - -public class TestPhysicalPlanning { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestPhysicalPlanning.class); - - @Test - public void testSimpleQuery(final DrillbitContext bitContext) throws Exception{ - - final DrillConfig c = DrillConfig.create(); - new NonStrictExpectations() { - { - bitContext.getMetrics(); - result = new MetricRegistry(); - bitContext.getAllocator(); - result = new TopLevelAllocator(); - bitContext.getConfig(); - result = c; - } - }; - - FunctionRegistry reg = new FunctionRegistry(c); - StoragePluginRegistry registry = new StoragePluginRegistry(bitContext); - DrillSqlWorker worker = new DrillSqlWorker(registry.getSchemaFactory(), reg); - worker.getPhysicalPlan("select * from cp.`employee.json`"); - - } -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java index b5ad235..9020c1a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java @@ -399,7 +399,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, FileUtils.getResourceAsFile("/scan_json_test_7.json").toURI().toString(), - FileSystem.getLocal(new Configuration()), null, null); + FileSystem.getLocal(new Configuration()), null); MockOutputMutator mutator = new MockOutputMutator(); List<ValueVector> addFields = mutator.getAddFields(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java index f6a8aa4..f6a7d97 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java @@ -429,8 +429,7 @@ public class ParquetRecordReaderTest { FileSystem fs = new CachedSingleFileSystem(fileName); for(int i = 0; i < 25; i++){ ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs, - new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), new FieldReference("_MAP", - ExpressionPosition.UNKNOWN), columns); + new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns); TestOutputMutator mutator = new TestOutputMutator(); rr.setup(mutator); Stopwatch watch = new Stopwatch(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/exec/java-exec/src/test/resources/join/merge_join_empty_batch.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/join/merge_join_empty_batch.json b/exec/java-exec/src/test/resources/join/merge_join_empty_batch.json new file mode 100644 index 0000000..549cb83 --- /dev/null +++ b/exec/java-exec/src/test/resources/join/merge_join_empty_batch.json @@ -0,0 +1,47 @@ +{ + head:{ + type:"APACHE_DRILL_PHYSICAL", + version:"1", + generator:{ + type:"manual" + } + }, + graph:[ + { + @id:1, + pop:"mock-scan", + url: "http://source1.apache.org";, + entries:[ + {records: 0, types: [ + {name: "blue", type: "INT", mode: "REQUIRED"}, + {name: "red", type: "INT", mode: "REQUIRED"}, + {name: "green", type: "INT", mode: "REQUIRED"} + ]} + ] + }, + { + @id:2, + pop:"mock-scan", + url: "http://source2.apache.org";, + entries:[ + {records: 50, types: [ + {name: "blue1", type: "INT", mode: "REQUIRED"}, + {name: "red1", type: "INT", mode: "REQUIRED"}, + {name: "green1", type: "INT", mode: "REQUIRED"} + ]} + ] + }, + { + @id: 3, + right: 1, + left: 2, + pop: "merge-join", + join-conditions: [ {relationship: "==", left: "blue1", right: "blue"} ] + }, + { + @id: 4, + child: 3, + pop: "screen" + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 44eb1c1..97cf61d 100644 --- a/pom.xml +++ b/pom.xml @@ -594,7 +594,7 @@ <dependency> <groupId>net.hydromatic</groupId> <artifactId>optiq-core</artifactId> - <version>0.4.18</version> + <version>0.6-SNAPSHOT</version> <exclusions> <exclusion> <groupId>org.jgrapht</groupId> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sample-data/nationsMF/nation.parquet ---------------------------------------------------------------------- diff --git a/sample-data/nationsMF/nation.parquet b/sample-data/nationsMF/nation.parquet new file mode 100644 index 0000000..db65492 Binary files /dev/null and b/sample-data/nationsMF/nation.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sample-data/nationsMF/nation2.parquet ---------------------------------------------------------------------- diff --git a/sample-data/nationsMF/nation2.parquet b/sample-data/nationsMF/nation2.parquet new file mode 100644 index 0000000..db65492 Binary files /dev/null and b/sample-data/nationsMF/nation2.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sample-data/nationsSF/nation.parquet ---------------------------------------------------------------------- diff --git a/sample-data/nationsSF/nation.parquet b/sample-data/nationsSF/nation.parquet new file mode 100644 index 0000000..db65492 Binary files /dev/null and b/sample-data/nationsSF/nation.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sample-data/regionsMF/region.parquet ---------------------------------------------------------------------- diff --git a/sample-data/regionsMF/region.parquet b/sample-data/regionsMF/region.parquet new file mode 100644 index 0000000..ab8122c Binary files /dev/null and b/sample-data/regionsMF/region.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sample-data/regionsMF/region2.parquet ---------------------------------------------------------------------- diff --git a/sample-data/regionsMF/region2.parquet b/sample-data/regionsMF/region2.parquet new file mode 100644 index 0000000..ab8122c Binary files /dev/null and b/sample-data/regionsMF/region2.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sample-data/regionsSF/region.parquet ---------------------------------------------------------------------- diff --git a/sample-data/regionsSF/region.parquet b/sample-data/regionsSF/region.parquet new file mode 100644 index 0000000..ab8122c Binary files /dev/null and b/sample-data/regionsSF/region.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sqlparser/src/main/java/org/apache/drill/jdbc/ConnectionConfig.java ---------------------------------------------------------------------- diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/ConnectionConfig.java b/sqlparser/src/main/java/org/apache/drill/jdbc/ConnectionConfig.java deleted file mode 100644 index 5a81035..0000000 --- a/sqlparser/src/main/java/org/apache/drill/jdbc/ConnectionConfig.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.jdbc; - -import java.util.Properties; -import java.util.TimeZone; - -public class ConnectionConfig { - private final Properties props; - - public ConnectionConfig(Properties p){ - this.props = p; - } - - public boolean isLocal(){ - return "local".equals(props.getProperty("zk")); - } - public String getZookeeperConnectionString(){ - return props.getProperty("zk"); - } - - public TimeZone getTimeZone(){ - return TimeZone.getDefault(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnection.java ---------------------------------------------------------------------- diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnection.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnection.java index ab1259a..5434f3d 100644 --- a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnection.java +++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnection.java @@ -35,7 +35,7 @@ public interface DrillConnection extends Connection{ /** Returns a view onto this connection's configuration properties. Code * within Optiq should use this view rather than calling * {@link java.util.Properties#getProperty(String)}. */ - ConnectionConfig config(); + DrillConnectionConfig config(); public DrillClient getClient(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionConfig.java ---------------------------------------------------------------------- diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionConfig.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionConfig.java new file mode 100644 index 0000000..0c0989d --- /dev/null +++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionConfig.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.jdbc; + +import java.util.Properties; +import java.util.TimeZone; + +import net.hydromatic.avatica.ConnectionConfig; +import net.hydromatic.avatica.ConnectionConfigImpl; + +public class DrillConnectionConfig extends ConnectionConfigImpl { + private final Properties props; + + public DrillConnectionConfig(Properties p){ + super(p); + this.props = p; + } + + public boolean isLocal(){ + return "local".equals(props.getProperty("zk")); + } + public String getZookeeperConnectionString(){ + return props.getProperty("zk"); + } + + public TimeZone getTimeZone(){ + return TimeZone.getDefault(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionImpl.java ---------------------------------------------------------------------- diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionImpl.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionImpl.java index 11a1157..5dc0d58 100644 --- a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionImpl.java +++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillConnectionImpl.java @@ -45,7 +45,7 @@ import org.apache.drill.exec.server.RemoteServiceSet; */ abstract class DrillConnectionImpl extends AvaticaConnection implements org.apache.drill.jdbc.DrillConnection { public final DrillStatementRegistry registry = new DrillStatementRegistry(); - final ConnectionConfig config; + final DrillConnectionConfig config; static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConnection.class); @@ -57,7 +57,7 @@ abstract class DrillConnectionImpl extends AvaticaConnection implements org.apac protected DrillConnectionImpl(Driver driver, AvaticaFactory factory, String url, Properties info) throws SQLException{ super(driver, factory, url, info); - this.config = new ConnectionConfig(info); + this.config = new DrillConnectionConfig(info); this.allocator = new TopLevelAllocator(); @@ -90,7 +90,7 @@ abstract class DrillConnectionImpl extends AvaticaConnection implements org.apac } - public ConnectionConfig config(){ + public DrillConnectionConfig config(){ return config; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java ---------------------------------------------------------------------- diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java new file mode 100644 index 0000000..9977285 --- /dev/null +++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.jdbc.test; + +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; + +import org.apache.drill.common.util.TestTools; +import org.apache.drill.exec.store.hive.HiveTestDataGenerator; +import org.apache.drill.jdbc.Driver; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.common.base.Stopwatch; + +public class TestJdbcDistQuery { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcDistQuery.class); + + + // Set a timeout unless we're debugging. + @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(50000); + + private static final String WORKING_PATH; + static{ + Driver.load(); + WORKING_PATH = Paths.get("").toAbsolutePath().toString(); + + } + + @BeforeClass + public static void generateHive() throws Exception{ + new HiveTestDataGenerator().generateTestData(); + } + + + @Test + public void testSimpleQuerySingleFile() throws Exception{ + testQuery(String.format("select R_REGIONKEY, R_NAME " + + "from dfs.`%s/../sample-data/regionsSF/`", WORKING_PATH)); + } + + + @Test + public void testSimpleQueryMultiFile() throws Exception{ + testQuery(String.format("select R_REGIONKEY, R_NAME " + + "from dfs.`%s/../sample-data/regionsMF/`", WORKING_PATH)); + } + + @Test + public void testWhereOverSFile() throws Exception{ + testQuery(String.format("select R_REGIONKEY, R_NAME " + + "from dfs.`%s/../sample-data/regionsSF/` " + + "WHERE R_REGIONKEY = 1", WORKING_PATH)); + } + + @Test + public void testWhereOverMFile() throws Exception{ + testQuery(String.format("select R_REGIONKEY, R_NAME " + + "from dfs.`%s/../sample-data/regionsMF/` " + + "WHERE R_REGIONKEY = 1", WORKING_PATH)); + } + + + @Test + public void testAggSingleFile() throws Exception{ + testQuery(String.format("select R_REGIONKEY " + + "from dfs.`%s/../sample-data/regionsSF/` " + + "group by R_REGIONKEY", WORKING_PATH)); + } + + @Test + public void testAggMultiFile() throws Exception{ + testQuery(String.format("select R_REGIONKEY " + + "from dfs.`%s/../sample-data/regionsMF/` " + + "group by R_REGIONKEY", WORKING_PATH)); + } + + @Test + public void testAggOrderByDiffGKeyMultiFile() throws Exception{ + testQuery(String.format("select R_REGIONKEY, SUM(cast(R_REGIONKEY AS int)) As S " + + "from dfs.`%s/../sample-data/regionsMF/` " + + "group by R_REGIONKEY ORDER BY S", WORKING_PATH)); + } + + @Test + public void testAggOrderBySameGKeyMultiFile() throws Exception{ + testQuery(String.format("select R_REGIONKEY, SUM(cast(R_REGIONKEY AS int)) As S " + + "from dfs.`%s/../sample-data/regionsMF/` " + + "group by R_REGIONKEY " + + "ORDER BY R_REGIONKEY", WORKING_PATH)); + } + + @Test + public void testJoinSingleFile() throws Exception{ + testQuery(String.format("select T1.R_REGIONKEY " + + "from dfs.`%s/../sample-data/regionsSF/` as T1 " + + "join dfs.`%s/../sample-data/nationsSF/` as T2 " + + "on T1.R_REGIONKEY = T2.N_REGIONKEY", WORKING_PATH, WORKING_PATH)); + } + + @Test + public void testJoinMultiFile() throws Exception{ + testQuery(String.format("select T1.R_REGIONKEY " + + "from dfs.`%s/../sample-data/regionsMF/` as T1 " + + "join dfs.`%s/../sample-data/nationsMF/` as T2 " + + "on T1.R_REGIONKEY = T2.N_REGIONKEY", WORKING_PATH, WORKING_PATH)); + } + + @Test + public void testJoinMFileWhere() throws Exception{ + testQuery(String.format("select T1.R_REGIONKEY, T1.R_NAME " + + "from dfs.`%s/../sample-data/regionsMF/` as T1 " + + "join dfs.`%s/../sample-data/nationsMF/` as T2 " + + "on T1.R_REGIONKEY = T2.N_REGIONKEY " + + "WHERE T1.R_REGIONKEY = 3 ", WORKING_PATH, WORKING_PATH)); + } + + @Test + public void testSortSingleFile() throws Exception{ + testQuery(String.format("select R_REGIONKEY " + + "from dfs.`%s/../sample-data/regionsSF/` " + + "order by R_REGIONKEY", WORKING_PATH)); + } + + @Test + public void testSortMultiFile() throws Exception{ + testQuery(String.format("select R_REGIONKEY " + + "from dfs.`%s/../sample-data/regionsMF/` " + + "order by R_REGIONKEY", WORKING_PATH)); + } + + @Test + public void testSortMFileWhere() throws Exception{ + testQuery(String.format("select R_REGIONKEY " + + "from dfs.`%s/../sample-data/regionsMF/` " + + "WHERE R_REGIONKEY = 1 " + + "order by R_REGIONKEY ", WORKING_PATH )); + } + + @Test + public void testJoinAggSortWhere() throws Exception{ + testQuery(String.format("select T1.R_REGIONKEY, COUNT(1) as CNT " + + "from dfs.`%s/../sample-data/regionsMF/` as T1 " + + "join dfs.`%s/../sample-data/nationsMF/` as T2 " + + "on T1.R_REGIONKEY = T2.N_REGIONKEY " + + "WHERE T1.R_REGIONKEY = 3 " + + "GROUP BY T1.R_REGIONKEY " + + "ORDER BY T1.R_REGIONKEY",WORKING_PATH, WORKING_PATH )); + } + + @Test + public void testSelectLimit() throws Exception{ + testQuery(String.format("select R_REGIONKEY, R_NAME " + + "from dfs.`%s/../sample-data/regionsMF/` " + + "limit 2", WORKING_PATH)); + } + + private void testQuery(String sql) throws Exception{ + boolean success = false; + try (Connection c = DriverManager.getConnection("jdbc:drill:zk=local", null);) { + for (int x = 0; x < 1; x++) { + Stopwatch watch = new Stopwatch().start(); + Statement s = c.createStatement(); + ResultSet r = s.executeQuery(sql); + boolean first = true; + while (r.next()) { + ResultSetMetaData md = r.getMetaData(); + if (first == true) { + for (int i = 1; i <= md.getColumnCount(); i++) { + System.out.print(md.getColumnName(i)); + System.out.print('\t'); + } + System.out.println(); + first = false; + } + + for (int i = 1; i <= md.getColumnCount(); i++) { + System.out.print(r.getObject(i)); + System.out.print('\t'); + } + System.out.println(); + } + + System.out.println(String.format("Query completed in %d millis.", watch.elapsed(TimeUnit.MILLISECONDS))); + } + + System.out.println("\n\n\n"); + success = true; + }finally{ + if(!success) Thread.sleep(2000); + } + + + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6b517daa/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java ---------------------------------------------------------------------- diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java index 36bbc51..c067ae0 100644 --- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java +++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java @@ -41,7 +41,7 @@ public class TestJdbcQuery { // Set a timeout unless we're debugging. - @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(20000); + @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(200000000); private static final String WORKING_PATH; static{ @@ -56,6 +56,7 @@ public class TestJdbcQuery { } @Test + @Ignore public void testHiveRead() throws Exception{ testQuery("select * from hive.kv"); } @@ -67,6 +68,7 @@ public class TestJdbcQuery { } @Test + @Ignore public void testJsonQuery() throws Exception{ testQuery("select * from cp.`employee.json`"); } @@ -86,11 +88,13 @@ public class TestJdbcQuery { } @Test + @Ignore public void testWorkspace() throws Exception{ testQuery(String.format("select * from dfs.home.`%s/../sample-data/region.parquet`", WORKING_PATH)); } @Test + @Ignore public void testWildcard() throws Exception{ testQuery(String.format("select * from dfs.`%s/../sample-data/region.parquet`", WORKING_PATH)); } @@ -106,16 +110,19 @@ public class TestJdbcQuery { } @Test + @Ignore public void testLogicalExplain() throws Exception{ testQuery(String.format("EXPLAIN PLAN WITHOUT IMPLEMENTATION FOR select * from dfs.`%s/../sample-data/region.parquet`", WORKING_PATH)); } @Test + @Ignore public void testPhysicalExplain() throws Exception{ testQuery(String.format("EXPLAIN PLAN FOR select * from dfs.`%s/../sample-data/region.parquet`", WORKING_PATH)); } @Test + @Ignore public void checkUnknownColumn() throws Exception{ testQuery(String.format("SELECT unknownColumn FROM dfs.`%s/../sample-data/region.parquet`", WORKING_PATH)); }
