Yingyi Bu has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1584
Change subject: Add query parameter validation.
......................................................................
Add query parameter validation.
- Automatically adjust query parameters such as compiler.joinmemory
compiler.sortmemory, compiler.groupmemory if they're too small;
- Validate if query parameters specified in a query are supported.
Change-Id: Ia9e2df274a0c5cf598da6c37d0241b6b9d35c84d
---
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/FuzzyUtils.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/unsupported_parameter/unsupported_parameter.1.query.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/AsterixException.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
7 files changed, 79 insertions(+), 13 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/84/1584/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/FuzzyUtils.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/FuzzyUtils.java
index 42a9c25..f888ee0 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/FuzzyUtils.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/FuzzyUtils.java
@@ -40,8 +40,8 @@
private final static float JACCARD_DEFAULT_SIM_THRESHOLD = .8f;
private final static int EDIT_DISTANCE_DEFAULT_SIM_THRESHOLD = 1;
- private final static String SIM_FUNCTION_PROP_NAME = "simfunction";
- private final static String SIM_THRESHOLD_PROP_NAME = "simthreshold";
+ public final static String SIM_FUNCTION_PROP_NAME = "simfunction";
+ public final static String SIM_THRESHOLD_PROP_NAME = "simthreshold";
public final static String JACCARD_FUNCTION_NAME = "jaccard";
public final static String EDIT_DISTANCE_FUNCTION_NAME = "edit-distance";
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 0759599..4ec2e2a 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -35,9 +35,10 @@
import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExternalProperties;
import org.apache.asterix.common.config.OptimizationConfUtil;
-import org.apache.hyracks.control.common.config.OptionTypes;
import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.utils.Job;
import org.apache.asterix.common.utils.Job.SubmissionMode;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
@@ -47,8 +48,10 @@
import
org.apache.asterix.dataflow.data.common.MergeAggregationExpressionFactory;
import org.apache.asterix.dataflow.data.common.MissableTypeComputer;
import org.apache.asterix.dataflow.data.common.PartialAggregationTypeComputer;
+import org.apache.asterix.external.feed.watch.FeedActivityDetails;
import org.apache.asterix.formats.base.IDataFormat;
import org.apache.asterix.jobgen.QueryLogicalExpressionJobGen;
+import org.apache.asterix.lang.aql.statement.SubscribeFeedStatement;
import org.apache.asterix.lang.common.base.IAstPrintVisitorFactory;
import org.apache.asterix.lang.common.base.IQueryRewriter;
import org.apache.asterix.lang.common.base.IReturningStatement;
@@ -57,14 +60,16 @@
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.optimizer.base.FuzzyUtils;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.runtime.utils.AppContextInfo;
import
org.apache.asterix.transaction.management.service.transaction.JobIdFactory;
+import org.apache.asterix.translator.SessionConfig;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.asterix.translator.IStatementExecutor.Stats;
import org.apache.asterix.utils.ResourceUtils;
-import org.apache.asterix.translator.SessionConfig;
import
org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import
org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -74,12 +79,12 @@
import org.apache.hyracks.algebricks.compiler.api.ICompilerFactory;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.ExpressionRuntimeProvider;
import
org.apache.hyracks.algebricks.core.algebra.expressions.IConflictingTypeResolver;
import
org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
import
org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
import
org.apache.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
import
org.apache.hyracks.algebricks.core.algebra.expressions.IMissableTypeComputer;
-import
org.apache.hyracks.algebricks.core.algebra.expressions.ExpressionRuntimeProvider;
import
org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
import
org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPlotter;
@@ -90,18 +95,33 @@
import org.apache.hyracks.api.client.IClusterInfoCollector;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.client.NodeControllerInfo;
+import org.apache.hyracks.api.config.IOptionType;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.control.common.config.OptionTypes;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hyracks.api.config.IOptionType;
+import com.google.common.collect.ImmutableSet;
/**
* Provides helper methods for compilation of a query into a JobSpec and
submission
* to Hyracks through the Hyracks client interface.
*/
public class APIFramework {
+
+ private static final int MIN_FRAME_LIMIT_FOR_SORT_AND_GROUP_BY = 3;
+
+ private static final int MIN_FRAME_LIMIT_FOR_JOIN = 5;
+
+ // A white list of supported configurable parameters.
+ private static final Set<String> configurableParameterNames =
ImmutableSet.of(
+ CompilerProperties.COMPILER_JOINMEMORY_KEY,
CompilerProperties.COMPILER_GROUPMEMORY_KEY,
+ CompilerProperties.COMPILER_SORTMEMORY_KEY,
CompilerProperties.COMPILER_PARALLELISM_KEY,
+ FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
FuzzyUtils.SIM_FUNCTION_PROP_NAME,
+ FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
SubscribeFeedStatement.WAIT_FOR_COMPLETION,
+ FeedActivityDetails.FEED_POLICY_NAME,
FeedActivityDetails.COLLECT_LOCATIONS, "inline_with", "hash_merge",
+ "output-record-type");
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
@@ -215,12 +235,13 @@
CompilerProperties compilerProperties =
AppContextInfo.INSTANCE.getCompilerProperties();
int frameSize = compilerProperties.getFrameSize();
Map<String, String> querySpecificConfig = metadataProvider.getConfig();
+ validateConfig(querySpecificConfig); // Validates the user-overridden
query parameters.
int sortFrameLimit =
getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_SORTMEMORY_KEY),
- compilerProperties.getSortMemorySize(), frameSize);
+ compilerProperties.getSortMemorySize(), frameSize, false);
int groupFrameLimit =
getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY),
- compilerProperties.getGroupMemorySize(), frameSize);
+ compilerProperties.getGroupMemorySize(), frameSize, false);
int joinFrameLimit =
getFrameLimit(querySpecificConfig.get(CompilerProperties.COMPILER_JOINMEMORY_KEY),
- compilerProperties.getJoinMemorySize(), frameSize);
+ compilerProperties.getJoinMemorySize(), frameSize, true);
OptimizationConfUtil.getPhysicalOptimizationConfig().setFrameSize(frameSize);
OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalSort(sortFrameLimit);
OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalGroupBy(groupFrameLimit);
@@ -450,11 +471,14 @@
}
// Gets the frame limit.
- private int getFrameLimit(String parameter, long memBudgetInConfiguration,
int frameSize) {
+ private int getFrameLimit(String parameter, long memBudgetInConfiguration,
int frameSize, boolean joinOp)
+ throws AlgebricksException {
IOptionType<Long> longBytePropertyInterpreter =
OptionTypes.LONG_BYTE_UNIT;
long memBudget =
parameter == null ? memBudgetInConfiguration :
longBytePropertyInterpreter.parse(parameter);
- return (int) (memBudget / frameSize);
+ int frameLimit = (int) (memBudget / frameSize);
+ int minframeLimit = joinOp ? MIN_FRAME_LIMIT_FOR_JOIN :
MIN_FRAME_LIMIT_FOR_SORT_AND_GROUP_BY;
+ return Math.max(frameLimit, minframeLimit);
}
// Gets the parallelism parameter.
@@ -462,4 +486,12 @@
IOptionType<Integer> integerIPropertyInterpreter = OptionTypes.INTEGER;
return parameter == null ? parallelismInConfiguration :
integerIPropertyInterpreter.parse(parameter);
}
+
+ private void validateConfig(Map<String, String> config) throws
AlgebricksException {
+ for (String parameterName : config.keySet()) {
+ if (!configurableParameterNames.contains(parameterName)) {
+ throw
AsterixException.create(ErrorCode.COMPILATION_UNSUPPORTED_QUERY_PARAMETER,
parameterName);
+ }
+ }
+ }
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/unsupported_parameter/unsupported_parameter.1.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/unsupported_parameter/unsupported_parameter.1.query.sqlpp
new file mode 100644
index 0000000..b0c83c4
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/unsupported_parameter/unsupported_parameter.1.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+SET `compiler.joinmem` "4MB"
+
+SELECT 1;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index d26e0d5..40c8c80 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3276,6 +3276,12 @@
</compilation-unit>
</test-case>
<test-case FilePath="misc">
+ <compilation-unit name="unsupported_parameter">
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Query parameter compiler.joinmem is not
supported</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
<compilation-unit name="uuid">
<output-dir compare="Text">uuid</output-dir>
</compilation-unit>
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/AsterixException.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/AsterixException.java
index 7a73705..7cb7d8a 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/AsterixException.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/AsterixException.java
@@ -18,9 +18,9 @@
*/
package org.apache.asterix.common.exceptions;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-
import java.io.Serializable;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
public class AsterixException extends AlgebricksException {
private static final long serialVersionUID = 1L;
@@ -34,6 +34,10 @@
}
+ public static AsterixException create(int errorCode, Serializable...
params) {
+ return new AsterixException(errorCode, params);
+ }
+
public AsterixException(Throwable cause) {
super(cause);
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 8f859c7..9717fe5 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -85,6 +85,7 @@
public static final int COMPILATION_AQLPLUS_NO_SUCH_JOIN_TYPE = 1025;
public static final int COMPILATION_FUNC_EXPRESSION_CANNOT_UTILIZE_INDEX =
1026;
public static final int
COMPILATION_DATASET_TYPE_DOES_NOT_HAVE_PRIMARY_INDEX = 1027;
+ public static final int COMPILATION_UNSUPPORTED_QUERY_PARAMETER = 1028;
// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index bf9c6f9..55c4de5 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -71,6 +71,7 @@
1025 = There is no such join type in AQL+
1026 = The given function expression %1$s cannot utilize index
1027 = Dataset of type %1$s doesn't have a primary index
+1028 = Query parameter %1$s is not supported
# Feed Errors
3001 = Illegal state.
--
To view, visit https://asterix-gerrit.ics.uci.edu/1584
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia9e2df274a0c5cf598da6c37d0241b6b9d35c84d
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <[email protected]>
