Repository: incubator-drill Updated Branches: refs/heads/master 838fda824 -> ecaa838fe
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index bdbb642..ef1529c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -26,14 +26,18 @@ import net.hydromatic.optiq.tools.ValidationException; import org.apache.drill.common.logical.LogicalPlan; import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.common.BaseScreenRel; +import org.apache.drill.exec.planner.common.DrillStoreRel; import org.apache.drill.exec.planner.logical.DrillImplementor; import org.apache.drill.exec.planner.logical.DrillParseContext; import org.apache.drill.exec.planner.logical.DrillRel; import org.apache.drill.exec.planner.logical.DrillRuleSets; import org.apache.drill.exec.planner.logical.DrillScreenRel; -import org.apache.drill.exec.planner.logical.DrillStoreRel; +import org.apache.drill.exec.planner.physical.Prel; import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory; import org.eigenbase.rel.RelNode; +import org.eigenbase.relopt.RelTraitSet; import org.eigenbase.sql.SqlExplain; import org.eigenbase.sql.SqlKind; import org.eigenbase.sql.SqlLiteral; @@ -45,12 +49,25 @@ public class DrillSqlWorker { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSqlWorker.class); private final Planner planner; + private final static RuleSet[] RULES = new RuleSet[]{DrillRuleSets.DRILL_BASIC_RULES, DrillRuleSets.DRILL_PHYSICAL_MEM}; + private final static int LOGICAL_RULES = 0; + private final static int PHYSICAL_MEM_RULES = 1; public DrillSqlWorker(DrillSchemaFactory schemaFactory) throws Exception { this.planner = Frameworks.getPlanner(ConnectionConfig.Lex.MYSQL, schemaFactory, SqlStdOperatorTable.instance(), new RuleSet[]{DrillRuleSets.DRILL_BASIC_RULES}); } - public LogicalPlan getPlan(String sql) throws SqlParseException, ValidationException, RelConversionException{ + private class RelResult{ + final ResultMode mode; + final RelNode node; + public RelResult(ResultMode mode, RelNode node) { + super(); + this.mode = mode; + this.node = node; + } + } + + private RelResult getRel(String sql) throws SqlParseException, ValidationException, RelConversionException{ SqlNode sqlNode = planner.parse(sql); ResultMode resultMode = ResultMode.EXEC; @@ -72,13 +89,20 @@ public class DrillSqlWorker { SqlNode validatedNode = planner.validate(sqlNode); RelNode relNode = planner.convert(validatedNode); - RelNode convertedRelNode = planner.transform(0, planner.getEmptyTraitSet().plus(DrillRel.CONVENTION), relNode); + return new RelResult(resultMode, relNode); + } + + + + public LogicalPlan getLogicalPlan(String sql) throws SqlParseException, ValidationException, RelConversionException{ + RelResult result = getRel(sql); + RelNode convertedRelNode = planner.transform(LOGICAL_RULES, planner.getEmptyTraitSet().plus(DrillRel.DRILL_LOGICAL), result.node); if(convertedRelNode instanceof DrillStoreRel){ throw new UnsupportedOperationException(); }else{ convertedRelNode = new DrillScreenRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode); } - DrillImplementor implementor = new DrillImplementor(new DrillParseContext(), resultMode); + DrillImplementor implementor = new DrillImplementor(new DrillParseContext(), result.mode); implementor.go( (DrillRel) convertedRelNode); planner.close(); planner.reset(); @@ -87,4 +111,13 @@ public class DrillSqlWorker { } + public PhysicalPlan getPhysicalPlan(String sql) throws SqlParseException, ValidationException, RelConversionException{ + RelResult result = getRel(sql); + RelTraitSet traits = planner.getEmptyTraitSet().plus(Prel.DRILL_PHYSICAL); + RelNode transformed = planner.transform(PHYSICAL_MEM_RULES, traits, result.node); + planner.close(); + planner.reset(); + return null; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/planner/torel/ConversionContext.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/torel/ConversionContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/torel/ConversionContext.java index d4aabb4..470a33d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/torel/ConversionContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/torel/ConversionContext.java @@ -34,15 +34,15 @@ import org.apache.drill.common.logical.data.Project; import org.apache.drill.common.logical.data.Scan; import org.apache.drill.common.logical.data.Union; import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor; +import org.apache.drill.exec.planner.common.BaseFilterRel; +import org.apache.drill.exec.planner.common.DrillLimitRel; +import org.apache.drill.exec.planner.common.BaseProjectRel; +import org.apache.drill.exec.planner.common.BaseScanRel; +import org.apache.drill.exec.planner.common.DrillUnionRel; import org.apache.drill.exec.planner.logical.DrillAggregateRel; -import org.apache.drill.exec.planner.logical.DrillFilterRel; import org.apache.drill.exec.planner.logical.DrillJoinRel; -import org.apache.drill.exec.planner.logical.DrillLimitRel; -import org.apache.drill.exec.planner.logical.DrillProjectRel; import org.apache.drill.exec.planner.logical.DrillRel; -import org.apache.drill.exec.planner.logical.DrillScanRel; import org.apache.drill.exec.planner.logical.DrillSortRel; -import org.apache.drill.exec.planner.logical.DrillUnionRel; import org.apache.drill.exec.planner.logical.ScanFieldDeterminer; import org.apache.drill.exec.planner.logical.ScanFieldDeterminer.FieldList; import org.eigenbase.rel.InvalidRelException; @@ -90,7 +90,7 @@ public class ConversionContext implements ToRelContext { public RelTraitSet getLogicalTraits(){ RelTraitSet set = RelTraitSet.createEmpty(); - set.add(DrillRel.CONVENTION); + set.add(DrillRel.DRILL_LOGICAL); return set; } @@ -121,17 +121,17 @@ public class ConversionContext implements ToRelContext { @Override public RelNode visitScan(Scan scan, ConversionContext context){ - return DrillScanRel.convert(scan, context); + return BaseScanRel.convert(scan, context); } @Override public RelNode visitFilter(Filter filter, ConversionContext context) throws InvalidRelException{ - return DrillFilterRel.convert(filter, context); + return BaseFilterRel.convert(filter, context); } @Override public RelNode visitProject(Project project, ConversionContext context) throws InvalidRelException{ - return DrillProjectRel.convert(project, context); + return BaseProjectRel.convert(project, context); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java index 8baa72a..3081b46 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.Collections; import java.util.List; -import org.apache.drill.common.logical.data.Scan; +import org.apache.drill.common.JSONOptions; import org.apache.drill.exec.physical.base.AbstractGroupScan; public abstract class AbstractStoragePlugin implements StoragePlugin{ @@ -46,9 +46,10 @@ public abstract class AbstractStoragePlugin implements StoragePlugin{ } @Override - public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException { + public AbstractGroupScan getPhysicalScan(JSONOptions selection) throws IOException { throw new UnsupportedOperationException(); } - + + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java index 2e54b0d..1c986a9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java @@ -23,7 +23,8 @@ import java.util.List; import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaPlus; -import org.apache.drill.common.logical.data.Scan; +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.physical.base.AbstractGroupScan; public interface StoragePlugin { @@ -41,8 +42,9 @@ public interface StoragePlugin { * @return * @throws IOException */ - public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException; + public AbstractGroupScan getPhysicalScan(JSONOptions selection) throws IOException; public Schema createAndAddSchema(SchemaPlus parent); + public StoragePluginConfig getConfig(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java index d25d501..120fdf4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -25,9 +25,10 @@ import java.util.Map; import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaPlus; +import org.apache.drill.common.JSONOptions; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.logical.FormatPluginConfig; -import org.apache.drill.common.logical.data.Scan; +import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; @@ -75,11 +76,11 @@ public class FileSystemPlugin extends AbstractStoragePlugin{ List<WorkspaceSchemaFactory> factories = null; if(config.workspaces == null || config.workspaces.isEmpty()){ - factories = Collections.singletonList(new WorkspaceSchemaFactory("default", name, fs, "/", matchers)); + factories = Collections.singletonList(new WorkspaceSchemaFactory(this, "default", name, fs, "/", matchers)); }else{ factories = Lists.newArrayList(); for(Map.Entry<String, String> space : config.workspaces.entrySet()){ - factories.add(new WorkspaceSchemaFactory(space.getKey(), name, fs, space.getValue(), matchers)); + factories.add(new WorkspaceSchemaFactory(this, space.getKey(), name, fs, space.getValue(), matchers)); } } this.schemaFactory = new FileSystemSchemaFactory(name, factories); @@ -94,8 +95,13 @@ public class FileSystemPlugin extends AbstractStoragePlugin{ } @Override - public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException { - FormatSelection formatSelection = scan.getSelection().getWith(context.getConfig(), FormatSelection.class); + public StoragePluginConfig getConfig() { + return config; + } + + @Override + public AbstractGroupScan getPhysicalScan(JSONOptions selection) throws IOException { + FormatSelection formatSelection = selection.getWith(context.getConfig(), FormatSelection.class); FormatPlugin plugin; if(formatSelection.getFormat() instanceof NamedFormatPluginConfig){ plugin = formatsByName.get( ((NamedFormatPluginConfig) formatSelection.getFormat()).name); @@ -103,7 +109,7 @@ public class FileSystemPlugin extends AbstractStoragePlugin{ plugin = formatPluginsByConfig.get(formatSelection.getFormat()); } if(plugin == null) throw new IOException(String.format("Failure getting requested format plugin named '%s'. It was not one of the format plugins registered.", formatSelection.getFormat())); - return plugin.getGroupScan(scan.getOutputReference(), formatSelection.getSelection()); + return plugin.getGroupScan(formatSelection.getSelection()); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java index a37142e..73e414c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java @@ -39,7 +39,7 @@ public interface FormatPlugin { public FormatMatcher getMatcher(); - public AbstractGroupScan getGroupScan(FieldReference outputRef, FileSelection selection) throws IOException; + public AbstractGroupScan getGroupScan(FileSelection selection) throws IOException; public List<QueryOptimizerRule> getOptimizerRules(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java index c69edb7..2ce7648 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java @@ -43,10 +43,12 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa private final DrillFileSystem fs; private final String storageEngineName; private final String schemaName; + private final FileSystemPlugin plugin; - public WorkspaceSchemaFactory(String schemaName, String storageEngineName, DrillFileSystem fileSystem, String path, + public WorkspaceSchemaFactory(FileSystemPlugin plugin, String schemaName, String storageEngineName, DrillFileSystem fileSystem, String path, List<FormatMatcher> formatMatchers) throws ExecutionSetupException { this.fs = fileSystem; + this.plugin = plugin; this.root = new Path(path); this.fileMatchers = Lists.newArrayList(); this.dirMatchers = Lists.newArrayList(); @@ -76,7 +78,7 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa try { Object selection = m.isReadable(fileSelection); if (selection != null) - return new DynamicDrillTable(storageEngineName, selection, m.getFormatPlugin().getStorageConfig()); + return new DynamicDrillTable(plugin, storageEngineName, selection, m.getFormatPlugin().getStorageConfig()); } catch (IOException e) { logger.debug("File read failed.", e); } @@ -87,7 +89,7 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa for (FormatMatcher m : fileMatchers) { Object selection = m.isReadable(fileSelection); if (selection != null) - return new DynamicDrillTable(storageEngineName, selection, m.getFormatPlugin().getStorageConfig()); + return new DynamicDrillTable(plugin, storageEngineName, selection, m.getFormatPlugin().getStorageConfig()); } return null; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java index d7949c3..780ec14 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java @@ -92,21 +92,21 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements return blockSplittable; }; - public abstract RecordReader getRecordReader(FragmentContext context, FileWork fileWork, FieldReference ref, List<SchemaPath> columns) throws ExecutionSetupException; + public abstract RecordReader getRecordReader(FragmentContext context, FileWork fileWork, List<SchemaPath> columns) throws ExecutionSetupException; RecordBatch getBatch(FragmentContext context, EasySubScan scan) throws ExecutionSetupException { List<RecordReader> readers = Lists.newArrayList(); for(FileWork work : scan.getWorkUnits()){ - readers.add(getRecordReader(context, work, scan.getRef(), scan.getColumns())); + readers.add(getRecordReader(context, work, scan.getColumns())); } return new ScanBatch(context, readers.iterator()); } @Override - public AbstractGroupScan getGroupScan(FieldReference outputRef, FileSelection selection) throws IOException { - return new EasyGroupScan(selection, this, outputRef, null); + public AbstractGroupScan getGroupScan(FileSelection selection) throws IOException { + return new EasyGroupScan(selection, this, null); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java index a7f556e..6015865 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java @@ -55,7 +55,6 @@ public class EasyGroupScan extends AbstractGroupScan{ private final FileSelection selection; private final EasyFormatPlugin<?> formatPlugin; - private final FieldReference ref; private final int maxWidth; private final List<SchemaPath> columns; @@ -69,7 +68,6 @@ public class EasyGroupScan extends AbstractGroupScan{ @JsonProperty("storage") StoragePluginConfig storageConfig, // @JsonProperty("format") FormatPluginConfig formatConfig, // @JacksonInject StoragePluginRegistry engineRegistry, // - @JsonProperty("ref") FieldReference ref, // @JsonProperty("columns") List<SchemaPath> columns ) throws IOException, ExecutionSetupException { @@ -77,20 +75,17 @@ public class EasyGroupScan extends AbstractGroupScan{ Preconditions.checkNotNull(formatPlugin, "Unable to load format plugin for provided format config."); this.selection = new FileSelection(files, true); this.maxWidth = selection.getFileStatusList(formatPlugin.getFileSystem()).size(); - this.ref = ref; this.columns = columns; } public EasyGroupScan( FileSelection selection, // EasyFormatPlugin<?> formatPlugin, // - FieldReference ref, // List<SchemaPath> columns ) throws IOException{ this.selection = selection; this.maxWidth = selection.getFileStatusList(formatPlugin.getFileSystem()).size(); this.formatPlugin = formatPlugin; - this.ref = ref; this.columns = columns; } @@ -161,7 +156,7 @@ public class EasyGroupScan extends AbstractGroupScan{ Preconditions.checkArgument(!filesForMinor.isEmpty(), String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId)); - return new EasySubScan(convert(filesForMinor), formatPlugin, ref, columns); + return new EasySubScan(convert(filesForMinor), formatPlugin, columns); } private List<FileWorkImpl> convert(List<CompleteFileWork> list){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java index 72d1fe6..6631a6a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java @@ -42,7 +42,6 @@ public class EasySubScan extends AbstractSubScan{ private final List<FileWorkImpl> files; private final EasyFormatPlugin<?> formatPlugin; - private final FieldReference ref; private final List<SchemaPath> columns; @JsonCreator @@ -51,21 +50,18 @@ public class EasySubScan extends AbstractSubScan{ @JsonProperty("storage") StoragePluginConfig storageConfig, // @JsonProperty("format") FormatPluginConfig formatConfig, // @JacksonInject StoragePluginRegistry engineRegistry, // - @JsonProperty("ref") FieldReference ref, // @JsonProperty("columns") List<SchemaPath> columns // ) throws IOException, ExecutionSetupException { this.formatPlugin = (EasyFormatPlugin<?>) engineRegistry.getFormatPlugin(storageConfig, formatConfig); Preconditions.checkNotNull(this.formatPlugin); this.files = files; - this.ref = ref; this.columns = columns; } - public EasySubScan(List<FileWorkImpl> files, EasyFormatPlugin<?> plugin, FieldReference ref, List<SchemaPath> columns){ + public EasySubScan(List<FileWorkImpl> files, EasyFormatPlugin<?> plugin, List<SchemaPath> columns){ this.formatPlugin = plugin; this.files = files; - this.ref = ref; this.columns = columns; } @@ -89,11 +85,6 @@ public class EasySubScan extends AbstractSubScan{ return formatPlugin.getConfig(); } - @JsonProperty("ref") - public FieldReference getRef() { - return ref; - } - @JsonProperty("columns") public List<SchemaPath> getColumns(){ return columns; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java index 82bf3bf..807c67e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java @@ -45,9 +45,9 @@ public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> { } @Override - public RecordReader getRecordReader(FragmentContext context, FileWork fileWork, FieldReference ref, + public RecordReader getRecordReader(FragmentContext context, FileWork fileWork, List<SchemaPath> columns) throws ExecutionSetupException { - return new JSONRecordReader(context, fileWork.getPath(), this.getFileSystem().getUnderlying(), ref, columns); + return new JSONRecordReader(context, fileWork.getPath(), this.getFileSystem().getUnderlying(), columns); } @JsonTypeName("json") http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java index d327b77..ac03444 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java @@ -80,23 +80,21 @@ public class JSONRecordReader implements RecordReader { private OutputMutator outputMutator; private BufferAllocator allocator; private int batchSize; - private final FieldReference ref; private final List<SchemaPath> columns; public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, int batchSize, - FieldReference ref, List<SchemaPath> columns) { + List<SchemaPath> columns) { this.hadoopPath = new Path(inputPath); this.fileSystem = fileSystem; this.allocator = fragmentContext.getAllocator(); this.batchSize = batchSize; valueVectorMap = Maps.newHashMap(); - this.ref = ref; this.columns = columns; } - public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, FieldReference ref, + public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, List<SchemaPath> columns) { - this(fragmentContext, inputPath, fileSystem, DEFAULT_LENGTH, ref, columns); + this(fragmentContext, inputPath, fileSystem, DEFAULT_LENGTH, columns); } private JsonParser getParser() { @@ -149,7 +147,6 @@ public class JSONRecordReader implements RecordReader { // Garbage collect fields never referenced in this batch for (Field field : Iterables.concat(currentSchema.removeUnreadFields(), removedFields)) { diffSchema.addRemovedField(field); - outputMutator.removeField(field.getAsMaterializedField(ref)); } if (diffSchema.isChanged()) { @@ -510,7 +507,7 @@ public class JSONRecordReader implements RecordReader { } private VectorHolder getOrCreateVectorHolder(Field field) throws SchemaChangeException { - String fullFieldName = ref != null ? ref.getPath() + "." + field.getFullFieldName() : field.getFullFieldName(); + String fullFieldName = field.getFullFieldName(); VectorHolder holder = valueVectorMap.get(fullFieldName); if (holder == null) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java index a1d575c..46d85a5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java @@ -22,8 +22,8 @@ import java.io.IOException; import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaPlus; +import org.apache.drill.common.JSONOptions; import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.logical.data.Scan; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; import org.apache.drill.exec.store.hive.schema.HiveSchemaFactory; @@ -37,7 +37,6 @@ public class HiveStoragePlugin extends AbstractStoragePlugin { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveStoragePlugin.class); private final HiveStoragePluginConfig config; - private final HiveConf hiveConf; private final HiveSchemaFactory schemaFactory; private final DrillbitContext context; private final String name; @@ -45,8 +44,7 @@ public class HiveStoragePlugin extends AbstractStoragePlugin { public HiveStoragePlugin(HiveStoragePluginConfig config, DrillbitContext context, String name) throws ExecutionSetupException { this.config = config; this.context = context; - this.schemaFactory = new HiveSchemaFactory(config, name, config.getHiveConf()); - this.hiveConf = config.getHiveConf(); + this.schemaFactory = new HiveSchemaFactory(this, name, config.getHiveConf()); this.name = name; } @@ -63,8 +61,8 @@ public class HiveStoragePlugin extends AbstractStoragePlugin { } @Override - public HiveScan getPhysicalScan(Scan scan) throws IOException { - HiveReadEntry hiveReadEntry = scan.getSelection().getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){}); + public HiveScan getPhysicalScan(JSONOptions selection) throws IOException { + HiveReadEntry hiveReadEntry = selection.getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){}); try { return new HiveScan(hiveReadEntry, this, null); } catch (ExecutionSetupException e) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java index abec2c5..7d6bc72 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.planner.logical.DrillTable; import org.apache.drill.exec.store.hive.HiveReadEntry; +import org.apache.drill.exec.store.hive.HiveStoragePlugin; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -38,8 +39,8 @@ public class DrillHiveTable extends DrillTable{ private final Table hiveTable; - public DrillHiveTable(String storageEngineName, HiveReadEntry readEntry, StoragePluginConfig storageEngineConfig) { - super(storageEngineName, readEntry, storageEngineConfig); + public DrillHiveTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) { + super(storageEngineName, plugin, readEntry); this.hiveTable = new org.apache.hadoop.hive.ql.metadata.Table(readEntry.getTable()); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java index 091381f..4e80afc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java @@ -33,6 +33,7 @@ import org.apache.drill.exec.store.SchemaFactory; import org.apache.drill.exec.store.SchemaHolder; import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema; import org.apache.drill.exec.store.hive.HiveReadEntry; +import org.apache.drill.exec.store.hive.HiveStoragePlugin; import org.apache.drill.exec.store.hive.HiveStoragePluginConfig; import org.apache.drill.exec.store.hive.HiveTable; import org.apache.hadoop.hive.conf.HiveConf; @@ -58,12 +59,12 @@ public class HiveSchemaFactory implements SchemaFactory { private LoadingCache<String, List<String>> databases; private LoadingCache<String, List<String>> tableNameLoader; private LoadingCache<String, LoadingCache<String, HiveReadEntry>> tableLoaders; - private HiveStoragePluginConfig pluginConfig; + private HiveStoragePlugin plugin; private final String schemaName; - public HiveSchemaFactory(HiveStoragePluginConfig pluginConfig, String name, HiveConf hiveConf) throws ExecutionSetupException { + public HiveSchemaFactory(HiveStoragePlugin plugin, String name, HiveConf hiveConf) throws ExecutionSetupException { this.schemaName = name; - this.pluginConfig = pluginConfig; + this.plugin = plugin; try { this.mClient = new HiveMetaStoreClient(hiveConf); @@ -255,7 +256,7 @@ public class HiveSchemaFactory implements SchemaFactory { DrillTable getDrillTable(String dbName, String t){ HiveReadEntry entry = getSelectionBaseOnName(dbName, t); if(entry == null) return null; - return new DrillHiveTable(schemaName, entry, pluginConfig); + return new DrillHiveTable(schemaName, plugin, entry); } HiveReadEntry getSelectionBaseOnName(String dbName, String t) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java index 0578b06..0d1635f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java @@ -23,7 +23,8 @@ import java.util.ArrayList; import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaPlus; -import org.apache.drill.common.logical.data.Scan; +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; @@ -35,14 +36,16 @@ import com.fasterxml.jackson.databind.ObjectMapper; public class MockStorageEngine extends AbstractStoragePlugin { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class); + private final MockStorageEngineConfig configuration; + public MockStorageEngine(MockStorageEngineConfig configuration, DrillbitContext context, String name) { - + this.configuration = configuration; } @Override - public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException { + public AbstractGroupScan getPhysicalScan(JSONOptions selection) throws IOException { - ArrayList<MockScanEntry> readEntries = scan.getSelection().getListWith(new ObjectMapper(), + ArrayList<MockScanEntry> readEntries = selection.getListWith(new ObjectMapper(), new TypeReference<ArrayList<MockScanEntry>>() { }); @@ -54,5 +57,10 @@ public class MockStorageEngine extends AbstractStoragePlugin { return null; } + @Override + public StoragePluginConfig getConfig() { + return configuration; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java index bfaaa45..cde9b08 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java @@ -100,8 +100,8 @@ public class ParquetFormatPlugin implements FormatPlugin{ } @Override - public ParquetGroupScan getGroupScan(FieldReference outputRef, FileSelection selection) throws IOException { - return new ParquetGroupScan( selection.getFileStatusList(fs), this, outputRef); + public ParquetGroupScan getGroupScan(FileSelection selection) throws IOException { + return new ParquetGroupScan( selection.getFileStatusList(fs), this); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java index 185fc0d..47218fe 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java @@ -87,7 +87,6 @@ public class ParquetGroupScan extends AbstractGroupScan { private final ParquetFormatPlugin formatPlugin; private final ParquetFormatConfig formatConfig; private final FileSystem fs; - private final FieldReference ref; private List<EndpointAffinity> endpointAffinities; private List<SchemaPath> columns; @@ -107,7 +106,6 @@ public class ParquetGroupScan extends AbstractGroupScan { @JsonProperty("storage") StoragePluginConfig storageConfig, // @JsonProperty("format") FormatPluginConfig formatConfig, // @JacksonInject StoragePluginRegistry engineRegistry, // - @JsonProperty("ref") FieldReference ref, // @JsonProperty("columns") List<SchemaPath> columns // ) throws IOException, ExecutionSetupException { engineRegistry.init(DrillConfig.create()); @@ -120,14 +118,12 @@ public class ParquetGroupScan extends AbstractGroupScan { this.fs = formatPlugin.getFileSystem().getUnderlying(); this.formatConfig = formatPlugin.getConfig(); this.entries = entries; - this.ref = ref; this.readFooterFromEntries(); } public ParquetGroupScan(List<FileStatus> files, // - ParquetFormatPlugin formatPlugin, // - FieldReference ref) // + ParquetFormatPlugin formatPlugin) // throws IOException { this.formatPlugin = formatPlugin; this.columns = null; @@ -139,7 +135,6 @@ public class ParquetGroupScan extends AbstractGroupScan { entries.add(new ReadEntryWithPath(file.getPath().toString())); } - this.ref = ref; readFooter(files); } @@ -282,7 +277,7 @@ public class ParquetGroupScan extends AbstractGroupScan { Preconditions.checkArgument(!rowGroupsForMinor.isEmpty(), String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId)); - return new ParquetRowGroupScan(formatPlugin, convertToReadEntries(rowGroupsForMinor), ref, columns); + return new ParquetRowGroupScan(formatPlugin, convertToReadEntries(rowGroupsForMinor), columns); } @@ -296,10 +291,7 @@ public class ParquetGroupScan extends AbstractGroupScan { } return entries; } - - public FieldReference getRef() { - return ref; - } + @Override public int getMaxParallelizationWidth() { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java index 9e1cc66..a3c57cf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java @@ -72,7 +72,6 @@ class ParquetRecordReader implements RecordReader { private int bitWidthAllFixedFields; private boolean allFieldsFixedLength; private int recordsPerBatch; - private final FieldReference ref; private long totalRecords; private long rowGroupOffset; @@ -93,22 +92,24 @@ class ParquetRecordReader implements RecordReader { int rowGroupIndex; - public ParquetRecordReader(FragmentContext fragmentContext, - String path, int rowGroupIndex, FileSystem fs, - CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref, + public ParquetRecordReader(FragmentContext fragmentContext, // + String path, // + int rowGroupIndex, // + FileSystem fs, // + CodecFactoryExposer codecFactoryExposer, // + ParquetMetadata footer, // List<SchemaPath> columns) throws ExecutionSetupException { - this(fragmentContext, DEFAULT_BATCH_LENGTH_IN_BITS, path, rowGroupIndex, fs, codecFactoryExposer, footer, ref, + this(fragmentContext, DEFAULT_BATCH_LENGTH_IN_BITS, path, rowGroupIndex, fs, codecFactoryExposer, footer, columns); } public ParquetRecordReader(FragmentContext fragmentContext, long batchSize, String path, int rowGroupIndex, FileSystem fs, - CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref, + CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, List<SchemaPath> columns) throws ExecutionSetupException { this.allocator = fragmentContext.getAllocator(); hadoopPath = new Path(path); fileSystem = fs; - this.ref = ref; this.codecFactoryExposer = codecFactoryExposer; this.rowGroupIndex = rowGroupIndex; this.batchSize = batchSize; @@ -255,11 +256,7 @@ class ParquetRecordReader implements RecordReader { } private SchemaPath toFieldName(String[] paths) { - if(this.ref == null){ - return new SchemaPath(Joiner.on('/').join(paths), ExpressionPosition.UNKNOWN); - }else{ - return ref.getChild(paths); - } + return new SchemaPath(Joiner.on('/').join(paths), ExpressionPosition.UNKNOWN); } private TypeProtos.DataMode getDataMode(ColumnDescriptor column) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java index 0e672d0..0b1a788 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java @@ -52,7 +52,6 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan { public final ParquetFormatConfig formatConfig; private final ParquetFormatPlugin formatPlugin; private final List<RowGroupReadEntry> rowGroupReadEntries; - private final FieldReference ref; private final List<SchemaPath> columns; @JsonCreator @@ -61,7 +60,6 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan { @JsonProperty("storage") StoragePluginConfig storageConfig, // @JsonProperty("format") FormatPluginConfig formatConfig, // @JsonProperty("entries") LinkedList<RowGroupReadEntry> rowGroupReadEntries, // - @JsonProperty("ref") FieldReference ref, // @JsonProperty("columns") List<SchemaPath> columns // ) throws ExecutionSetupException { @@ -72,19 +70,16 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan { Preconditions.checkNotNull(formatPlugin); this.rowGroupReadEntries = rowGroupReadEntries; this.formatConfig = formatPlugin.getConfig(); - this.ref = ref; this.columns = columns; } public ParquetRowGroupScan( // ParquetFormatPlugin formatPlugin, // List<RowGroupReadEntry> rowGroupReadEntries, // - FieldReference ref, // List<SchemaPath> columns) { this.formatPlugin = formatPlugin; this.formatConfig = formatPlugin.getConfig(); this.rowGroupReadEntries = rowGroupReadEntries; - this.ref = ref; this.columns = columns; } @@ -103,10 +98,6 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan { return null; } - public FieldReference getRef() { - return ref; - } - @Override public Size getSize() { return null; @@ -130,7 +121,7 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan { @Override public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException { Preconditions.checkArgument(children.isEmpty()); - return new ParquetRowGroupScan(formatPlugin, rowGroupReadEntries, ref, columns); + return new ParquetRowGroupScan(formatPlugin, rowGroupReadEntries, columns); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java index 17e7da2..d36dbc0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java @@ -78,7 +78,6 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan context, e.getPath(), e.getRowGroupIndex(), fs, rowGroupScan.getStorageEngine().getCodecFactoryExposer(), footers.get(e.getPath()), - rowGroupScan.getRef(), rowGroupScan.getColumns() ) ); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index 08cb599..90b2a4d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -346,23 +346,6 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{ private void runSQL(String sql) { try{ - DrillSqlWorker sqlWorker = new DrillSqlWorker(context.getFactory()); - LogicalPlan plan = sqlWorker.getPlan(sql); - - - if(plan.getProperties().resultMode == ResultMode.LOGICAL){ - returnLogical(plan); - return; - } - - PhysicalPlan physical = convert(plan); - - if(plan.getProperties().resultMode == ResultMode.PHYSICAL){ - returnPhysical(physical); - return; - } - - runPhysicalPlan(physical); }catch(Exception e){ fail("Failure while parsing sql.", e); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/test/java/org/apache/drill/exec/planner/physical/TestPhysicalPlanning.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/physical/TestPhysicalPlanning.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/physical/TestPhysicalPlanning.java new file mode 100644 index 0000000..68d1eef --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/physical/TestPhysicalPlanning.java @@ -0,0 +1,39 @@ +package org.apache.drill.exec.planner.physical; + +import mockit.NonStrictExpectations; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.expression.FunctionRegistry; +import org.apache.drill.exec.memory.TopLevelAllocator; +import org.apache.drill.exec.planner.sql.DrillSqlWorker; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.junit.Test; + +import com.codahale.metrics.MetricRegistry; + +public class TestPhysicalPlanning { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestPhysicalPlanning.class); + + @Test + public void testSimpleQuery(final DrillbitContext bitContext) throws Exception{ + + final DrillConfig c = DrillConfig.create(); + new NonStrictExpectations() { + { + bitContext.getMetrics(); + result = new MetricRegistry(); + bitContext.getAllocator(); + result = new TopLevelAllocator(); + bitContext.getConfig(); + result = c; + } + }; + + FunctionRegistry reg = new FunctionRegistry(c); + StoragePluginRegistry registry = new StoragePluginRegistry(bitContext); + DrillSqlWorker worker = new DrillSqlWorker(registry.getSchemaFactory(), reg); + worker.getPhysicalPlan("select * from cp.`employee.json`"); + + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java index e812997..b5ad235 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java @@ -139,7 +139,7 @@ public class JSONRecordReaderTest { }; JSONRecordReader jr = new JSONRecordReader(context, FileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString(), - FileSystem.getLocal(new Configuration()), null, null); + FileSystem.getLocal(new Configuration()), null); MockOutputMutator mutator = new MockOutputMutator(); List<ValueVector> addFields = mutator.getAddFields(); @@ -169,7 +169,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(), - FileSystem.getLocal(new Configuration()), null, null); + FileSystem.getLocal(new Configuration()), null); MockOutputMutator mutator = new MockOutputMutator(); List<ValueVector> addFields = mutator.getAddFields(); @@ -211,7 +211,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(), FileSystem.getLocal(new Configuration()), - 64, null, Arrays.asList(new SchemaPath("test", ExpressionPosition.UNKNOWN))); // batch only fits 1 int + 64, Arrays.asList(new SchemaPath("test", ExpressionPosition.UNKNOWN))); // batch only fits 1 int MockOutputMutator mutator = new MockOutputMutator(); List<ValueVector> addFields = mutator.getAddFields(); List<MaterializedField> removedFields = mutator.getRemovedFields(); @@ -246,7 +246,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(), FileSystem.getLocal(new Configuration()), - 64, null, null); // batch only fits 1 int + 64, null); // batch only fits 1 int MockOutputMutator mutator = new MockOutputMutator(); List<ValueVector> addFields = mutator.getAddFields(); List<MaterializedField> removedFields = mutator.getRemovedFields(); @@ -304,7 +304,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, FileUtils.getResourceAsFile("/scan_json_test_3.json").toURI().toString(), - FileSystem.getLocal(new Configuration()), null, null); + FileSystem.getLocal(new Configuration()), null); MockOutputMutator mutator = new MockOutputMutator(); List<ValueVector> addFields = mutator.getAddFields(); @@ -333,7 +333,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, FileUtils.getResourceAsFile("/scan_json_test_4.json").toURI().toString(), - FileSystem.getLocal(new Configuration()), null, null); + FileSystem.getLocal(new Configuration()), null); MockOutputMutator mutator = new MockOutputMutator(); List<ValueVector> addFields = mutator.getAddFields(); @@ -366,7 +366,7 @@ public class JSONRecordReaderTest { JSONRecordReader jr = new JSONRecordReader(context, FileUtils.getResourceAsFile("/scan_json_test_5.json").toURI().toString(), - FileSystem.getLocal(new Configuration()), null, null); + FileSystem.getLocal(new Configuration()), null); MockOutputMutator mutator = new MockOutputMutator(); List<ValueVector> addFields = mutator.getAddFields(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/exec/java-exec/src/test/resources/storage-engines.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/storage-engines.json b/exec/java-exec/src/test/resources/storage-engines.json index 6e4d23e..73899ee 100644 --- a/exec/java-exec/src/test/resources/storage-engines.json +++ b/exec/java-exec/src/test/resources/storage-engines.json @@ -3,10 +3,11 @@ dfs: { type: "file", connection: "file:///" - }, + }, cp: { type: "file", connection: "classpath:///" - } - } + } + + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/22c41907/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java ---------------------------------------------------------------------- diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java index 8cd682b..1e4edca 100644 --- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java +++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java @@ -184,7 +184,6 @@ public class JdbcTest { // Assert.assertTrue(config != null && config instanceof QueueRSE.QueueRSEConfig); Scan scan = findOnlyOperator(plan, Scan.class); Assert.assertEquals("donuts-json", scan.getStorageEngine()); - Assert.assertEquals("_MAP", scan.getOutputReference().getPath()); Project project = findOnlyOperator(plan, Project.class); Assert.assertEquals(1, project.getSelections().length); Assert.assertEquals(Scan.class, project.getInput().getClass()); @@ -240,7 +239,6 @@ public class JdbcTest { // Assert.assertTrue(config != null && config instanceof QueueRSE.QueueRSEConfig); Scan scan = findOnlyOperator(plan, Scan.class); Assert.assertEquals("donuts-json", scan.getStorageEngine()); - Assert.assertEquals("_MAP", scan.getOutputReference().getPath()); Filter filter = findOnlyOperator(plan, Filter.class); Assert.assertTrue(filter.getInput() instanceof Scan); Project[] projects = Iterables.toArray(findOperator(plan, Project.class), Project.class);
