DRILL-45: Implement builders for Scan, Sort, LogicalPlan and PlanProperties.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/727adb75 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/727adb75 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/727adb75 Branch: refs/heads/master Commit: 727adb75bf6098b25bf7596faa026bd181b51e53 Parents: 438ae02 Author: Ben Becker <[email protected]> Authored: Tue Jun 11 13:09:29 2013 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Wed Oct 30 17:21:25 2013 -0700 ---------------------------------------------------------------------- .../org/apache/drill/common/PlanProperties.java | 14 ++-- .../drill/common/logical/LogicalPlan.java | 7 +- .../apache/drill/common/logical/data/Scan.java | 6 +- .../apache/drill/common/logical/data/Store.java | 22 +++--- .../src/test/resources/storage_engine_plan.json | 48 ++++++------ .../drill/exec/physical/PhysicalPlan.java | 2 +- .../common/logical/LogicalPlanBuilder.java | 47 ++++++++++++ .../drill/common/logical/PlanProperties.java | 81 ++++++++++++++++++++ .../drill/common/logical/data/ScanBuilder.java | 49 ++++++++++++ .../drill/common/logical/data/StoreBuilder.java | 49 ++++++++++++ .../common/logical/LogicalPlanBuilderTest.java | 64 ++++++++++++++++ .../common/logical/data/ScanBuilderTest.java | 48 ++++++++++++ .../common/logical/data/StoreBuilderTest.java | 48 ++++++++++++ 13 files changed, 440 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/common/src/main/java/org/apache/drill/common/PlanProperties.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/PlanProperties.java b/common/src/main/java/org/apache/drill/common/PlanProperties.java index 637dd66..ecab468 100644 --- a/common/src/main/java/org/apache/drill/common/PlanProperties.java +++ b/common/src/main/java/org/apache/drill/common/PlanProperties.java @@ -26,11 +26,11 @@ public class PlanProperties { public PlanType type; public int version; - public Generator generator = new Generator(); - - @JsonInclude(Include.NON_NULL) - public static class Generator{ - public String type; - public String info; - } + public Generator generator = new Generator(); + + @JsonInclude(Include.NON_NULL) + public static class Generator{ + public String type; + public String info; + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java index 868f9ba..8be44e9 100644 --- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java +++ b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java @@ -22,7 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.drill.common.PlanProperties; +import org.apache.drill.common.logical.PlanProperties; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.graph.Graph; import org.apache.drill.common.graph.GraphAlgos; @@ -104,4 +104,9 @@ public class LogicalPlan { throw new RuntimeException(e); } } + + public static LogicalPlanBuilder builder() { + return new LogicalPlanBuilder(); + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/common/src/main/java/org/apache/drill/common/logical/data/Scan.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Scan.java b/common/src/main/java/org/apache/drill/common/logical/data/Scan.java index 9c315ff..294c0e8 100644 --- a/common/src/main/java/org/apache/drill/common/logical/data/Scan.java +++ b/common/src/main/java/org/apache/drill/common/logical/data/Scan.java @@ -59,8 +59,8 @@ public class Scan extends SourceOperator{ return logicalVisitor.visitScan(this, value); } - - - + public static ScanBuilder builder() { + return new ScanBuilder(); + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/common/src/main/java/org/apache/drill/common/logical/data/Store.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Store.java b/common/src/main/java/org/apache/drill/common/logical/data/Store.java index bc34cc7..99d1d50 100644 --- a/common/src/main/java/org/apache/drill/common/logical/data/Store.java +++ b/common/src/main/java/org/apache/drill/common/logical/data/Store.java @@ -56,16 +56,18 @@ public class Store extends SinkOperator{ return partition; } - @Override - public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E { - return logicalVisitor.visitStore(this, value); - } - - @Override - public Iterator<LogicalOperator> iterator() { - return Iterators.singletonIterator(getInput()); - } - + @Override + public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E { + return logicalVisitor.visitStore(this, value); + } + @Override + public Iterator<LogicalOperator> iterator() { + return Iterators.singletonIterator(getInput()); + } + public static StoreBuilder builder() { + return new StoreBuilder(); + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/common/src/test/resources/storage_engine_plan.json ---------------------------------------------------------------------- diff --git a/common/src/test/resources/storage_engine_plan.json b/common/src/test/resources/storage_engine_plan.json index efde80d..6be9deb 100644 --- a/common/src/test/resources/storage_engine_plan.json +++ b/common/src/test/resources/storage_engine_plan.json @@ -1,30 +1,32 @@ { - head:{ - type:"APACHE_DRILL_LOGICAL", - version:"1", - generator:{ - type:"manual", - info:"na" + "head" : { + "type" : "APACHE_DRILL_LOGICAL", + "version" : 1, + "generator" : { + "type" : "manual", + "info" : "na" } }, - storage:{ - mock-engine: { - type:"mock", - url: "http://www.apache.org/"; - } + "storage" : { + "mock-engine": { + type : "mock", + url : "http://www.apache.org/"; + } }, - query:[ - { - @id:"1", - op:"scan", - storageengine:"mock-engine", - selection: {} - }, + "query" : [ { - input: 1, - op: "store", - storageengine:"mock-engine", - target: {} + "op" : "scan", + "@id" : 1, + "storageengine" : "mock-engine", + "selection" : null, + "ref" : null + }, { + "op" : "store", + "@id" : 2, + "input" : 1, + "target" : null, + "partition" : null, + "storageEngine" : "mock-engine" } ] -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/exec/java-exec/src/main/java/org/apache/drill/exec/physical/PhysicalPlan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/PhysicalPlan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/PhysicalPlan.java index 7349dad..4082661 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/PhysicalPlan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/PhysicalPlan.java @@ -20,7 +20,7 @@ package org.apache.drill.exec.physical; import java.io.IOException; import java.util.List; -import org.apache.drill.common.PlanProperties; +import org.apache.drill.common.logical.PlanProperties; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.graph.Graph; import org.apache.drill.common.graph.GraphAlgos; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java new file mode 100644 index 0000000..675a5c8 --- /dev/null +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.logical; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.drill.common.logical.data.LogicalOperator; + +/** + * A programmatic builder for logical plans. + */ +public class LogicalPlanBuilder { + private PlanProperties planProperties; + private ImmutableMap.Builder<String, StorageEngineConfig> storageEngines = ImmutableMap.builder(); + private ImmutableList.Builder<LogicalOperator> operators = ImmutableList.builder(); + + public LogicalPlanBuilder planProperties(PlanProperties planProperties) { + this.planProperties = planProperties; + return this; + } + public LogicalPlanBuilder addStorageEngine(String name, StorageEngineConfig config) { + this.storageEngines.put(name, config); + return this; + } + public LogicalPlanBuilder addLogicalOperator(LogicalOperator operator) { + this.operators.add(operator); + return this; + } + public LogicalPlan build() { + return new LogicalPlan(this.planProperties, this.storageEngines.build(), this.operators.build()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java new file mode 100644 index 0000000..996b932 --- /dev/null +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/PlanProperties.java @@ -0,0 +1,81 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.logical; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Logical plan meta properties. + */ +public class PlanProperties { + public static enum PlanType {APACHE_DRILL_LOGICAL, APACHE_DRILL_PHYSICAL} + + public PlanType type; + public int version; + public Generator generator; + + @JsonInclude(Include.NON_NULL) + public static class Generator{ + public String type; + public String info; + + private Generator(@JsonProperty("type") String type, @JsonProperty("info") String info) { + this.type = type; + this.info = info; + } + } + + private PlanProperties(@JsonProperty("version") int version, + @JsonProperty("generator") Generator generator, + @JsonProperty("type") PlanType type) { + this.version = version; + this.generator = generator; + this.type = type; + } + + public static PlanPropertiesBuilder builder() { + return new PlanPropertiesBuilder(); + } + + public static class PlanPropertiesBuilder { + private int version; + private Generator generator; + private PlanType type; + + public PlanPropertiesBuilder type(PlanType type) { + this.type = type; + return this; + } + + public PlanPropertiesBuilder version(int version) { + this.version = version; + return this; + } + + public PlanPropertiesBuilder generator(String type, String info) { + this.generator = new Generator(type, info); + return this; + } + + public PlanProperties build() { + return new PlanProperties(version, generator, type); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/ScanBuilder.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/ScanBuilder.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/ScanBuilder.java new file mode 100644 index 0000000..3eaa91f --- /dev/null +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/ScanBuilder.java @@ -0,0 +1,49 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.logical.data; + +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.expression.FieldReference; + +/** + * Builder for the scan operator + */ +public class ScanBuilder { + private String storageEngine; + private JSONOptions selection; + private FieldReference outputReference; + + public ScanBuilder storageEngine(String storageEngine) { + this.storageEngine = storageEngine; + return this; + } + + public ScanBuilder selection(JSONOptions selection) { + this.selection = selection; + return this; + } + + public ScanBuilder outputReference(FieldReference outputReference) { + this.outputReference = outputReference; + return this; + } + + public Scan build() { + return new Scan(storageEngine, selection, outputReference); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/StoreBuilder.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/StoreBuilder.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/StoreBuilder.java new file mode 100644 index 0000000..09d20b6 --- /dev/null +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/StoreBuilder.java @@ -0,0 +1,49 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.logical.data; + +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.defs.PartitionDef; + +/** + * Builder for the scan operator + */ +public class StoreBuilder { + private String storageEngine; + private JSONOptions target; + private PartitionDef partition; + + public StoreBuilder storageEngine(String storageEngine) { + this.storageEngine = storageEngine; + return this; + } + + public StoreBuilder target(JSONOptions target) { + this.target = target; + return this; + } + + public StoreBuilder partition(PartitionDef partition) { + this.partition = partition; + return this; + } + + public Store build() { + return new Store(storageEngine, target, partition); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/LogicalPlanBuilderTest.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/LogicalPlanBuilderTest.java b/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/LogicalPlanBuilderTest.java new file mode 100644 index 0000000..efe946b --- /dev/null +++ b/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/LogicalPlanBuilderTest.java @@ -0,0 +1,64 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.logical; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.logical.data.Scan; +import org.apache.drill.common.logical.data.Store; +import org.apache.drill.common.util.FileUtils; +import org.apache.drill.storage.MockStorageEngineConfig; +import org.junit.Test; + +import java.io.IOException; + +import static junit.framework.TestCase.assertEquals; + +public class LogicalPlanBuilderTest { + + + /** + * Tests assembling the same plan as simple_engine_plan.json + */ + @Test + public void testBuildSimplePlan() throws IOException { + + PlanProperties planProperties = PlanProperties.builder() + .generator("manual", "na") + .version(1) + .type(PlanProperties.PlanType.APACHE_DRILL_LOGICAL) + .build(); + + Scan scan = new Scan("mock-engine", null, null); + Store store = new Store("mock-engine", null, null); + store.setInput(scan); + + LogicalPlanBuilder builder = LogicalPlan.builder() + .planProperties(planProperties) + .addStorageEngine("mock-engine", new MockStorageEngineConfig("http://www.apache.org/";)) + .addLogicalOperator(scan) + .addLogicalOperator(store); + + LogicalPlan fromBuilder = builder.build(); + + DrillConfig config = DrillConfig.create(); + LogicalPlan fromJson = LogicalPlan.parse(config, FileUtils.getResourceAsString("/storage_engine_plan.json")); + + assertEquals(fromJson.toJsonString(config), fromBuilder.toJsonString(config)); + + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/data/ScanBuilderTest.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/data/ScanBuilderTest.java b/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/data/ScanBuilderTest.java new file mode 100644 index 0000000..b318b3d --- /dev/null +++ b/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/data/ScanBuilderTest.java @@ -0,0 +1,48 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.logical.data; + +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.expression.FieldReference; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ScanBuilderTest{ + + /** + * Build a Scan operator and validate each field + */ + @Test + public void testBuild() { + FieldReference outputFieldReference = new FieldReference("Flavour"); + JSONOptions selection = null; + String storageEngine = "mock-storage"; + + Scan scanOp = Scan.builder() + .storageEngine(storageEngine) + .outputReference(outputFieldReference) + .selection(selection) + .build(); + + assertEquals(scanOp.getOutputReference(), outputFieldReference); + assertEquals(scanOp.getSelection(), selection); + assertEquals(scanOp.getStorageEngine(), storageEngine); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/727adb75/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/data/StoreBuilderTest.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/data/StoreBuilderTest.java b/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/data/StoreBuilderTest.java new file mode 100644 index 0000000..73f0402 --- /dev/null +++ b/sandbox/prototype/common/src/test/java/org/apache/drill/common/logical/data/StoreBuilderTest.java @@ -0,0 +1,48 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.common.logical.data; + +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.defs.PartitionDef; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class StoreBuilderTest{ + + /** + * Build a Store operator and validate each field + */ + @Test + public void testBuild() { + String storageEngine = "mock-storage"; + PartitionDef partition = new PartitionDef(PartitionDef.PartitionType.RANGE, null, null); + JSONOptions target = null; + + Store storeOp = Store.builder() + .storageEngine(storageEngine) + .partition(partition) + .target(target) + .build(); + + assertEquals(storeOp.getStorageEngine(), storageEngine); + assertEquals(storeOp.getPartition(), partition); + assertEquals(storeOp.getTarget(), target); + } + +} \ No newline at end of file
