>From Ali Alsuliman <[email protected]>: Ali Alsuliman has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21070?usp=email )
Change subject: [ASTERIXDB-3749][COMP] Update names and default values of compiler properties ...................................................................... [ASTERIXDB-3749][COMP] Update names and default values of compiler properties - user model changes: yes - storage format changes: no - interface changes: no Ext-ref: MB-71173 Change-Id: I6836c0fe036be1a02904474ff8e6913c1a2ba24c Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21070 Integration-Tests: Jenkins <[email protected]> Tested-by: Ali Alsuliman <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java 9 files changed, 47 insertions(+), 29 deletions(-) Approvals: Jenkins: Verified Ali Alsuliman: Looks good to me, approved; Verified diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java index e53a846..be313aa 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java @@ -99,11 +99,12 @@ CompilerProperties.COMPILER_DISJUNCTION_HASH_THRESHOLD, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, CompilerProperties.COMPILER_MAX_VARIABLE_OCCURRENCES_INLINING_KEY, CompilerProperties.COMPILER_DELTALAKE_FILESPLITS_KEY, - CompilerProperties.COMPILER_MAX_EXPRESSION_TREE_SIZE_KEY, FuzzyUtils.SIM_FUNCTION_PROP_NAME, - FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION, - FeedActivityDetails.FEED_POLICY_NAME, FeedActivityDetails.COLLECT_LOCATIONS, - SqlppQueryRewriter.INLINE_WITH_OPTION, SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, - "hash_merge", "output-record-type", DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION, + CompilerProperties.COMPILER_OPTIMIZE_EXPRESSION_MAX_ARGS_KEY, FuzzyUtils.SIM_FUNCTION_PROP_NAME, + CompilerProperties.COMPILER_EXTRACT_COMMON_EXPRESSION_LIMIT_KEY, FuzzyUtils.SIM_THRESHOLD_PROP_NAME, + StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME, + FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION, + SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, "hash_merge", "output-record-type", + DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION, SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN, EquivalenceClassUtils.REWRITE_INTERNAL_QUERYUID_PK, SqlppQueryRewriter.SQL_COMPAT_OPTION, JoinEnum.CBO_FULL_ENUM_LEVEL_KEY, JoinEnum.CBO_CP_ENUM_KEY)); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index c8c1535..edd93ae 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -46,25 +46,26 @@ "compiler.column.filter" : true, "compiler.copy.to.write.buffer.size" : 8388608, "compiler.deltalake.filesplits" : false, - "compiler.disjunction.hash.threshold" : -1, + "compiler.disjunction.as.join" : false, + "compiler.disjunction.hash.threshold" : 40, "compiler\.external\.field\.pushdown" : true, + "compiler.extract.common.expression.limit" : 100, "compiler.forcejoinorder" : false, "compiler\.framesize" : 32768, "compiler\.groupmemory" : 163840, "compiler\.indexonly" : true, "compiler\.internal\.sanitycheck" : true, "compiler\.joinmemory" : 262144, - "compiler\.max\.expression\.tree\.size" : 10000, "compiler\.max\.variable\.occurrences\.inlining" : 128, "compiler.min.groupmemory" : 524288, "compiler.min.joinmemory" : 524288, "compiler\.min\.memory\.allocation" : true, "compiler.min.sortmemory" : 524288, "compiler.min.windowmemory" : 524288, + "compiler\.optimize\.expression.max\.args" : 100, "compiler.ordered.fields" : false, "compiler\.parallelism" : 0, "compiler.queryplanshape" : "zigzag", - "compiler.rewrite.disjunction" : true, "compiler.runtime.memory.overhead" : 5, "compiler\.sort\.parallel" : false, "compiler\.sort\.samples" : 100, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index 161fda5..b616a93 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -46,25 +46,26 @@ "compiler.column.filter" : true, "compiler.copy.to.write.buffer.size" : 8388608, "compiler.deltalake.filesplits" : false, - "compiler.disjunction.hash.threshold" : -1, - "compiler\.external\.field\.pushdown" : true, + "compiler.disjunction.as.join" : true, + "compiler.disjunction.hash.threshold" : 40, + "compiler.external.field.pushdown" : true, + "compiler.extract.common.expression.limit" : 1000, "compiler.forcejoinorder" : false, "compiler\.framesize" : 32768, "compiler\.groupmemory" : 163840, "compiler\.indexonly" : true, "compiler\.internal\.sanitycheck" : false, "compiler\.joinmemory" : 262144, - "compiler\.max\.expression\.tree\.size" : 10000, "compiler\.max\.variable\.occurrences\.inlining" : 128, "compiler.min.groupmemory" : 524288, "compiler.min.joinmemory" : 524288, "compiler\.min\.memory\.allocation" : true, "compiler.min.sortmemory" : 524288, "compiler.min.windowmemory" : 524288, + "compiler.optimize.expression.max.args" : 1000, "compiler.ordered.fields" : false, "compiler\.parallelism" : -1, "compiler.queryplanshape" : "zigzag", - "compiler.rewrite.disjunction" : true, "compiler.runtime.memory.overhead" : 5, "compiler\.sort\.parallel" : true, "compiler\.sort\.samples" : 100, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index 0c7c0d1..e51150e5f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -46,25 +46,26 @@ "compiler.column.filter" : true, "compiler.copy.to.write.buffer.size" : 8388608, "compiler.deltalake.filesplits" : false, - "compiler.disjunction.hash.threshold" : -1, + "compiler.disjunction.as.join" : true, + "compiler.disjunction.hash.threshold" : 40, "compiler\.external\.field\.pushdown" : true, + "compiler.extract.common.expression.limit" : 1000, "compiler.forcejoinorder" : false, "compiler\.framesize" : 32768, "compiler\.groupmemory" : 163840, "compiler\.indexonly" : true, "compiler\.internal\.sanitycheck" : false, "compiler\.joinmemory" : 262144, - "compiler\.max\.expression\.tree\.size" : 10000, "compiler\.max\.variable\.occurrences\.inlining" : 128, "compiler.min.groupmemory" : 524288, "compiler.min.joinmemory" : 524288, "compiler\.min\.memory\.allocation" : true, "compiler.min.sortmemory" : 524288, "compiler.min.windowmemory" : 524288, + "compiler.optimize.expression.max.args" : 1000, "compiler.ordered.fields" : false, "compiler\.parallelism" : 3, "compiler.queryplanshape" : "zigzag", - "compiler.rewrite.disjunction" : true, "compiler.runtime.memory.overhead" : 5, "compiler\.sort\.parallel" : true, "compiler\.sort\.samples" : 100, diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java index 814ec9f..51f4cbe 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@ -157,21 +157,24 @@ getRangedIntegerType(0, Integer.MAX_VALUE), 128, "Maximum occurrences of a variable allowed in an expression for inlining"), - COMPILER_MAX_EXPRESSION_TREE_SIZE( + COMPILER_OPTIMIZE_EXPRESSION_MAX_ARGS( getRangedIntegerType(1, Integer.MAX_VALUE), AlgebricksConfig.MAX_EXPRESSION_TREE_SIZE_DEFAULT, - "Maximum number of OR/AND arguments before skipping costly O(N^2) optimization passes " - + "(e.g. constant folding, CSE) on large IN-list queries"), + "Avoid costly O(N^2) expression optimization passes (e.g. constant folding, CSE) on large IN-list queries when the number of OR/AND function arguments >= the specified value"), + COMPILER_EXTRACT_COMMON_EXPRESSION_LIMIT( + getRangedIntegerType(1, Integer.MAX_VALUE), + AlgebricksConfig.COMMON_EXPRESSION_LIMIT_DEFAULT, + "The maximum number of common expressions to extract"), COMPILER_ORDERED_FIELDS(BOOLEAN, AlgebricksConfig.ORDERED_FIELDS, "Enable/disable select order list"), COMPILER_DELTALAKE_FILESPLITS(BOOLEAN, false, "Enable/disable delta lake file splits"), - COMPILER_REWRITE_DISJUNCTION( + COMPILER_DISJUNCTION_AS_JOIN( BOOLEAN, AlgebricksConfig.REWRITE_DISJUNCTION_DEFAULT, - "Set the mode for rewriting disjunctions to joins in query plans"), + "Rewrite disjunctions to joins in query plans"), COMPILER_DISJUNCTION_HASH_THRESHOLD( getRangedIntegerType(-1, Integer.MAX_VALUE), AlgebricksConfig.HASH_BASED_OR_THRESHOLD_DEFAULT, - "The max number of disjunctions after which a hash-based approach is used for evaluating OR operation"); + "The number of disjunctions after which a hash-based approach is used for evaluating OR operation (-1 disables using the hash-based approach)"); private final IOptionType type; private final Object defaultValue; @@ -227,7 +230,7 @@ public static final String COMPILER_INDEXONLY_KEY = Option.COMPILER_INDEXONLY.ini(); - public static final String COMPILER_REWRITE_DISJUNCTION_KEY = Option.COMPILER_REWRITE_DISJUNCTION.ini(); + public static final String COMPILER_REWRITE_DISJUNCTION_KEY = Option.COMPILER_DISJUNCTION_AS_JOIN.ini(); public static final String COMPILER_DISJUNCTION_HASH_THRESHOLD = Option.COMPILER_DISJUNCTION_HASH_THRESHOLD.ini(); @@ -260,7 +263,11 @@ public static final String COMPILER_MAX_VARIABLE_OCCURRENCES_INLINING_KEY = Option.COMPILER_MAX_VARIABLE_OCCURRENCES_INLINING.ini(); - public static final String COMPILER_MAX_EXPRESSION_TREE_SIZE_KEY = Option.COMPILER_MAX_EXPRESSION_TREE_SIZE.ini(); + public static final String COMPILER_OPTIMIZE_EXPRESSION_MAX_ARGS_KEY = + Option.COMPILER_OPTIMIZE_EXPRESSION_MAX_ARGS.ini(); + + public static final String COMPILER_EXTRACT_COMMON_EXPRESSION_LIMIT_KEY = + Option.COMPILER_EXTRACT_COMMON_EXPRESSION_LIMIT.ini(); public static final String COMPILER_ORDERED_FIELDS_KEY = Option.COMPILER_ORDERED_FIELDS.ini(); @@ -332,7 +339,7 @@ } public boolean rewriteDisjunctionToJoin() { - return accessor.getBoolean(Option.COMPILER_REWRITE_DISJUNCTION); + return accessor.getBoolean(Option.COMPILER_DISJUNCTION_AS_JOIN); } public int getHashBasedORThreshold() { @@ -418,7 +425,7 @@ } public int getMaxExpressionTreeSize() { - return accessor.getInt(Option.COMPILER_MAX_EXPRESSION_TREE_SIZE); + return accessor.getInt(Option.COMPILER_OPTIMIZE_EXPRESSION_MAX_ARGS); } public boolean isDeltaLakeFileSplitsEnabled() { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java index 9f2ac87..37fedc0 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java @@ -246,7 +246,7 @@ private static int getMaxExpressionTreeSize(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig, SourceLocation sourceLoc) throws AsterixException { String valueInQuery = - (String) querySpecificConfig.get(CompilerProperties.COMPILER_MAX_EXPRESSION_TREE_SIZE_KEY); + (String) querySpecificConfig.get(CompilerProperties.COMPILER_OPTIMIZE_EXPRESSION_MAX_ARGS_KEY); try { return valueInQuery == null ? compilerProperties.getMaxExpressionTreeSize() : OptionTypes.POSITIVE_INTEGER.parse(valueInQuery); diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java index 484ee1a..b8a37b33 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java @@ -49,6 +49,7 @@ public static final boolean COLUMN_FILTER_DEFAULT = true; public static final boolean ORDERED_FIELDS = true; public static final int MAX_VARIABLE_OCCURRENCES_INLINING_DEFAULT = 128; - public static final int HASH_BASED_OR_THRESHOLD_DEFAULT = -1; - public static final int MAX_EXPRESSION_TREE_SIZE_DEFAULT = 10000; + public static final int HASH_BASED_OR_THRESHOLD_DEFAULT = 40; + public static final int MAX_EXPRESSION_TREE_SIZE_DEFAULT = 1000; + public static final int COMMON_EXPRESSION_LIMIT_DEFAULT = 1000; } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java index 340f626..b5933d7 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java @@ -65,6 +65,7 @@ private static final String MIN_WINDOW_FRAMES = "MIN_WINDOW_FRAMES"; private static final String MAX_VARIABLE_OCCURRENCES_INLINING = "MAX_VARIABLE_OCCURRENCES_INLINING"; private static final String MAX_EXPRESSION_TREE_SIZE = "MAX_EXPRESSION_TREE_SIZE"; + private static final String COMMON_EXPRESSION_LIMIT = "COMMON_EXPRESSION_LIMIT"; private static final String ORDER_FIELDS = "ORDERED_FIELDS"; @@ -416,6 +417,10 @@ return getInt(MAX_EXPRESSION_TREE_SIZE, AlgebricksConfig.MAX_EXPRESSION_TREE_SIZE_DEFAULT); } + public int getCommonExpressionLimit() { + return getInt(COMMON_EXPRESSION_LIMIT, AlgebricksConfig.COMMON_EXPRESSION_LIMIT_DEFAULT); + } + public void setMaxExpressionTreeSize(int maxExpressionTreeSize) { setInt(MAX_EXPRESSION_TREE_SIZE, maxExpressionTreeSize); } @@ -455,4 +460,5 @@ String value = properties.getProperty(property); return value == null ? defaultValue : value; } + } diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java index 764e8d0..639bc7c 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java @@ -357,7 +357,7 @@ // Guard against unbounded map growth for very large expression trees (e.g. IN with a // long list expands into O(N) OR nodes). Beyond the limit the map lookups themselves // become O(N) due to equals() chain traversal, making the whole pass O(N^2). - if (exprEqClassMap.size() < context.getPhysicalOptimizationConfig().getMaxExpressionTreeSize()) { + if (exprEqClassMap.size() < context.getPhysicalOptimizationConfig().getCommonExpressionLimit()) { exprEqClass = new ExprEquivalenceClass(op, exprRef); exprEqClassMap.put(expr, exprEqClass); } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21070?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: phoenix Gerrit-Change-Id: I6836c0fe036be1a02904474ff8e6913c1a2ba24c Gerrit-Change-Number: 21070 Gerrit-PatchSet: 6 Gerrit-Owner: Shahrzad Shirazi <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]>
