This is an automated email from the ASF dual-hosted git repository. rong pushed a commit to branch iotdb-1022-v2 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 76378252e3989c10ced6e3efa5ef4970f3898939 Author: SteveYurongSu <[email protected]> AuthorDate: Thu May 13 18:02:35 2021 +0800 refactor Planner and LogicalGenerator --- .../apache/iotdb/cluster/query/ClusterPlanner.java | 20 +-- .../main/java/org/apache/iotdb/db/qp/Planner.java | 173 ++++----------------- .../iotdb/db/qp/strategy/LogicalGenerator.java | 63 +++++++- .../qp/strategy/optimizer/ILogicalOptimizer.java | 2 +- 4 files changed, 97 insertions(+), 161 deletions(-) diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanner.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanner.java index ecbfb74..f4e9ec4 100644 --- a/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanner.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanner.java @@ -22,11 +22,9 @@ package org.apache.iotdb.cluster.query; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.qp.Planner; import org.apache.iotdb.db.qp.logical.Operator; -import org.apache.iotdb.db.qp.logical.crud.SFWOperator; import org.apache.iotdb.db.qp.physical.PhysicalPlan; -import org.apache.iotdb.db.qp.strategy.PhysicalGenerator; +import org.apache.iotdb.db.qp.strategy.LogicalGenerator; import org.apache.iotdb.db.qp.strategy.optimizer.ConcatPathOptimizer; -import org.apache.iotdb.db.query.control.QueryResourceManager; import java.time.ZoneId; @@ -36,19 +34,9 @@ public class ClusterPlanner extends Planner { @Override public PhysicalPlan parseSQLToPhysicalPlan(String sqlStr, ZoneId zoneId, int fetchSize) throws QueryProcessException { - Operator operator = logicalGenerator.generate(sqlStr, zoneId); - int maxDeduplicatedPathNum = - QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(fetchSize); - if (operator instanceof SFWOperator && ((SFWOperator) operator).isLastQuery()) { - // Dataset of last query actually has only three columns, so we shouldn't limit the path num - // while constructing logical plan - // To avoid overflowing because logicalOptimize function may do maxDeduplicatedPathNum + 1, we - // set it to Integer.MAX_VALUE - 1 - maxDeduplicatedPathNum = Integer.MAX_VALUE - 1; - } - operator = logicalOptimize(operator, maxDeduplicatedPathNum); - PhysicalGenerator physicalGenerator = new ClusterPhysicalGenerator(); - return physicalGenerator.transformToPhysicalPlan(operator, fetchSize); + Operator operator = LogicalGenerator.generate(sqlStr, zoneId); + operator = logicalOptimize(operator, fetchSize); + return new ClusterPhysicalGenerator().transformToPhysicalPlan(operator, fetchSize); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java index 840b04f..027468b 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java @@ -23,14 +23,9 @@ import org.apache.iotdb.db.exception.query.LogicalOperatorException; import org.apache.iotdb.db.exception.query.LogicalOptimizeException; import org.apache.iotdb.db.exception.query.PathNumOverLimitException; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.PartialPath; -import org.apache.iotdb.db.qp.constant.SQLConstant; import org.apache.iotdb.db.qp.logical.Operator; -import org.apache.iotdb.db.qp.logical.crud.BasicFunctionOperator; import org.apache.iotdb.db.qp.logical.crud.FilterOperator; -import org.apache.iotdb.db.qp.logical.crud.FromOperator; import org.apache.iotdb.db.qp.logical.crud.QueryOperator; -import org.apache.iotdb.db.qp.logical.crud.SelectOperator; import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.strategy.LogicalGenerator; import org.apache.iotdb.db.qp.strategy.PhysicalGenerator; @@ -38,106 +33,41 @@ import org.apache.iotdb.db.qp.strategy.optimizer.ConcatPathOptimizer; import org.apache.iotdb.db.qp.strategy.optimizer.DnfFilterOptimizer; import org.apache.iotdb.db.qp.strategy.optimizer.MergeSingleFilterOptimizer; import org.apache.iotdb.db.qp.strategy.optimizer.RemoveNotOptimizer; -import org.apache.iotdb.db.query.control.QueryResourceManager; -import org.apache.iotdb.db.query.expression.ResultColumn; -import org.apache.iotdb.db.query.expression.unary.TimeSeriesOperand; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.service.rpc.thrift.TSRawDataQueryReq; import java.time.ZoneId; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import static org.apache.iotdb.db.conf.IoTDBConstant.TIME; +import static org.apache.iotdb.db.qp.logical.Operator.OperatorType.QUERY; +import static org.apache.iotdb.db.qp.logical.Operator.OperatorType.QUERY_INDEX; /** provide a integration method for other user. */ public class Planner { - protected LogicalGenerator logicalGenerator; - public Planner() { - this.logicalGenerator = new LogicalGenerator(); - } - - @TestOnly - public PhysicalPlan parseSQLToPhysicalPlan(String sqlStr) throws QueryProcessException { - return parseSQLToPhysicalPlan(sqlStr, ZoneId.systemDefault(), 1024); + // do nothing } /** @param fetchSize this parameter only take effect when it is a query plan */ public PhysicalPlan parseSQLToPhysicalPlan(String sqlStr, ZoneId zoneId, int fetchSize) throws QueryProcessException { - Operator operator = logicalGenerator.generate(sqlStr, zoneId); - int maxDeduplicatedPathNum = - QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(fetchSize); - if (operator instanceof QueryOperator && ((QueryOperator) operator).isLastQuery()) { - // Dataset of last query actually has only three columns, so we shouldn't limit the path num - // while constructing logical plan - // To avoid overflowing because logicalOptimize function may do maxDeduplicatedPathNum + 1, we - // set it to Integer.MAX_VALUE - 1 - maxDeduplicatedPathNum = Integer.MAX_VALUE - 1; - } - operator = logicalOptimize(operator, maxDeduplicatedPathNum); - PhysicalGenerator physicalGenerator = new PhysicalGenerator(); - return physicalGenerator.transformToPhysicalPlan(operator, fetchSize); + // from SQL to logical operator + Operator operator = LogicalGenerator.generate(sqlStr, zoneId); + // optimize the logical operator + operator = logicalOptimize(operator, fetchSize); + // from logical operator to physical plan + return new PhysicalGenerator().transformToPhysicalPlan(operator, fetchSize); } - /** convert raw data query to physical plan directly */ public PhysicalPlan rawDataQueryReqToPhysicalPlan( TSRawDataQueryReq rawDataQueryReq, ZoneId zoneId) - throws QueryProcessException, IllegalPathException { - List<String> paths = rawDataQueryReq.getPaths(); - long startTime = rawDataQueryReq.getStartTime(); - long endTime = rawDataQueryReq.getEndTime(); - - // construct query operator and set its global time filter - QueryOperator queryOp = new QueryOperator(SQLConstant.TOK_QUERY); - FromOperator fromOp = new FromOperator(SQLConstant.TOK_FROM); - SelectOperator selectOp = new SelectOperator(SQLConstant.TOK_SELECT, zoneId); - - // iterate the path list and add it to from operator - for (String p : paths) { - PartialPath path = new PartialPath(p); - fromOp.addPrefixTablePath(path); - } - selectOp.addResultColumn(new ResultColumn(new TimeSeriesOperand(new PartialPath("")))); - - queryOp.setSelectOperator(selectOp); - queryOp.setFromOperator(fromOp); - - // set time filter operator - FilterOperator filterOp = new FilterOperator(SQLConstant.KW_AND); - PartialPath timePath = new PartialPath(TIME); - filterOp.setSinglePath(timePath); - Set<PartialPath> pathSet = new HashSet<>(); - pathSet.add(timePath); - filterOp.setIsSingle(true); - filterOp.setPathSet(pathSet); - - BasicFunctionOperator left = - new BasicFunctionOperator( - SQLConstant.GREATERTHANOREQUALTO, timePath, Long.toString(startTime)); - BasicFunctionOperator right = - new BasicFunctionOperator(SQLConstant.LESSTHAN, timePath, Long.toString(endTime)); - filterOp.addChildOperator(left); - filterOp.addChildOperator(right); - - queryOp.setFilterOperator(filterOp); - - int maxDeduplicatedPathNum = - QueryResourceManager.getInstance().getMaxDeduplicatedPathNum(rawDataQueryReq.fetchSize); - if (queryOp.isLastQuery()) { - // Dataset of last query actually has only three columns, so we shouldn't limit the path num - // while constructing logical plan - // To avoid overflowing because logicalOptimize function may do maxDeduplicatedPathNum + 1, we - // set it to Integer.MAX_VALUE - 1 - maxDeduplicatedPathNum = Integer.MAX_VALUE - 1; - } - QueryOperator op = (QueryOperator) logicalOptimize(queryOp, maxDeduplicatedPathNum); - - PhysicalGenerator physicalGenerator = new PhysicalGenerator(); - return physicalGenerator.transformToPhysicalPlan(op, rawDataQueryReq.fetchSize); + throws IllegalPathException, QueryProcessException { + // from TSRawDataQueryReq to logical operator + Operator operator = LogicalGenerator.generate(rawDataQueryReq, zoneId); + // optimize the logical operator + operator = logicalOptimize(operator, rawDataQueryReq.fetchSize); + // from logical operator to physical plan + return new PhysicalGenerator().transformToPhysicalPlan(operator, rawDataQueryReq.fetchSize); } /** @@ -147,51 +77,11 @@ public class Planner { * @return optimized logical operator * @throws LogicalOptimizeException exception in logical optimizing */ - protected Operator logicalOptimize(Operator operator, int maxDeduplicatedPathNum) + protected Operator logicalOptimize(Operator operator, int fetchSize) throws LogicalOperatorException, PathNumOverLimitException { - switch (operator.getType()) { - case AUTHOR: - case METADATA: - case SET_STORAGE_GROUP: - case DELETE_STORAGE_GROUP: - case CREATE_TIMESERIES: - case DELETE_TIMESERIES: - case ALTER_TIMESERIES: - case LOADDATA: - case INSERT: - case GRANT_WATERMARK_EMBEDDING: - case REVOKE_WATERMARK_EMBEDDING: - case TTL: - case LOAD_CONFIGURATION: - case SHOW: - case LOAD_FILES: - case REMOVE_FILE: - case MOVE_FILE: - case FLUSH: - case MERGE: - case TRACING: - case CLEAR_CACHE: - case NULL: - case SHOW_MERGE_STATUS: - case DELETE_PARTITION: - case CREATE_SCHEMA_SNAPSHOT: - case KILL: - case CREATE_FUNCTION: - case DROP_FUNCTION: - case CREATE_TRIGGER: - case DROP_TRIGGER: - case START_TRIGGER: - case STOP_TRIGGER: - case QUERY: - case DELETE: - case CREATE_INDEX: - case DROP_INDEX: - return operator; - case QUERY_INDEX: - return optimizeQueryOperator((QueryOperator) operator, maxDeduplicatedPathNum); - default: - throw new LogicalOperatorException(operator.getType().toString(), ""); - } + return operator.getType().equals(QUERY) || operator.getType().equals(QUERY_INDEX) + ? optimizeQueryOperator((QueryOperator) operator, fetchSize) + : operator; } /** @@ -201,27 +91,28 @@ public class Planner { * @return optimized query operator * @throws LogicalOptimizeException exception in query optimizing */ - private QueryOperator optimizeQueryOperator(QueryOperator root, int maxDeduplicatedPathNum) + private QueryOperator optimizeQueryOperator(QueryOperator root, int fetchSize) throws LogicalOperatorException, PathNumOverLimitException { - ConcatPathOptimizer concatPathOptimizer = getConcatPathOptimizer(); - root = (QueryOperator) concatPathOptimizer.transform(root, maxDeduplicatedPathNum); + root = (QueryOperator) getConcatPathOptimizer().transform(root, fetchSize); + FilterOperator filter = root.getFilterOperator(); if (filter == null) { return root; } - Set<PartialPath> pathSet = filter.getPathSet(); - RemoveNotOptimizer removeNot = new RemoveNotOptimizer(); - filter = removeNot.optimize(filter); - DnfFilterOptimizer dnf = new DnfFilterOptimizer(); - filter = dnf.optimize(filter); - MergeSingleFilterOptimizer merge = new MergeSingleFilterOptimizer(); - filter = merge.optimize(filter); + filter = new RemoveNotOptimizer().optimize(filter); + filter = new DnfFilterOptimizer().optimize(filter); + filter = new MergeSingleFilterOptimizer().optimize(filter); root.setFilterOperator(filter); - filter.setPathSet(pathSet); + return root; } protected ConcatPathOptimizer getConcatPathOptimizer() { return new ConcatPathOptimizer(); } + + @TestOnly + public PhysicalPlan parseSQLToPhysicalPlan(String sqlStr) throws QueryProcessException { + return parseSQLToPhysicalPlan(sqlStr, ZoneId.systemDefault(), 1024); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java index e4f5a67..51a5e72 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java @@ -18,10 +18,21 @@ */ package org.apache.iotdb.db.qp.strategy; +import org.apache.iotdb.db.exception.metadata.IllegalPathException; +import org.apache.iotdb.db.metadata.PartialPath; +import org.apache.iotdb.db.qp.constant.SQLConstant; import org.apache.iotdb.db.qp.logical.Operator; +import org.apache.iotdb.db.qp.logical.crud.BasicFunctionOperator; +import org.apache.iotdb.db.qp.logical.crud.FilterOperator; +import org.apache.iotdb.db.qp.logical.crud.FromOperator; +import org.apache.iotdb.db.qp.logical.crud.QueryOperator; +import org.apache.iotdb.db.qp.logical.crud.SelectOperator; import org.apache.iotdb.db.qp.sql.IoTDBSqlVisitor; import org.apache.iotdb.db.qp.sql.SqlBaseLexer; import org.apache.iotdb.db.qp.sql.SqlBaseParser; +import org.apache.iotdb.db.query.expression.ResultColumn; +import org.apache.iotdb.db.query.expression.unary.TimeSeriesOperand; +import org.apache.iotdb.service.rpc.thrift.TSRawDataQueryReq; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; @@ -31,13 +42,15 @@ import org.antlr.v4.runtime.misc.ParseCancellationException; import org.antlr.v4.runtime.tree.ParseTree; import java.time.ZoneId; +import java.util.HashSet; +import java.util.Set; + +import static org.apache.iotdb.db.conf.IoTDBConstant.TIME; /** LogicalGenerator. */ public class LogicalGenerator { - public LogicalGenerator() {} - - public Operator generate(String sql, ZoneId zoneId) throws ParseCancellationException { + public static Operator generate(String sql, ZoneId zoneId) throws ParseCancellationException { IoTDBSqlVisitor ioTDBSqlVisitor = new IoTDBSqlVisitor(); ioTDBSqlVisitor.setZoneId(zoneId); CharStream charStream1 = CharStreams.fromString(sql); @@ -67,4 +80,48 @@ public class LogicalGenerator { } return ioTDBSqlVisitor.visit(tree); } + + public static Operator generate(TSRawDataQueryReq rawDataQueryReq, ZoneId zoneId) + throws IllegalPathException { + // construct query operator and set its global time filter + QueryOperator queryOp = new QueryOperator(SQLConstant.TOK_QUERY); + FromOperator fromOp = new FromOperator(SQLConstant.TOK_FROM); + SelectOperator selectOp = new SelectOperator(SQLConstant.TOK_SELECT, zoneId); + + // iterate the path list and add it to from operator + for (String p : rawDataQueryReq.getPaths()) { + PartialPath path = new PartialPath(p); + fromOp.addPrefixTablePath(path); + } + selectOp.addResultColumn(new ResultColumn(new TimeSeriesOperand(new PartialPath("")))); + + queryOp.setSelectOperator(selectOp); + queryOp.setFromOperator(fromOp); + + // set time filter operator + FilterOperator filterOp = new FilterOperator(SQLConstant.KW_AND); + PartialPath timePath = new PartialPath(TIME); + filterOp.setSinglePath(timePath); + Set<PartialPath> pathSet = new HashSet<>(); + pathSet.add(timePath); + filterOp.setIsSingle(true); + filterOp.setPathSet(pathSet); + + BasicFunctionOperator left = + new BasicFunctionOperator( + SQLConstant.GREATERTHANOREQUALTO, + timePath, + Long.toString(rawDataQueryReq.getStartTime())); + BasicFunctionOperator right = + new BasicFunctionOperator( + SQLConstant.LESSTHAN, timePath, Long.toString(rawDataQueryReq.getEndTime())); + filterOp.addChildOperator(left); + filterOp.addChildOperator(right); + + queryOp.setFilterOperator(filterOp); + + return queryOp; + } + + private LogicalGenerator() {} } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java index 26ab0a5..d851f7e 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ILogicalOptimizer.java @@ -26,6 +26,6 @@ import org.apache.iotdb.db.qp.logical.Operator; @FunctionalInterface public interface ILogicalOptimizer { - Operator transform(Operator operator, int maxDeduplicatedPathNum) + Operator transform(Operator operator, int fetchSize) throws LogicalOptimizeException, PathNumOverLimitException; }
