>From Ali Alsuliman <[email protected]>:
Ali Alsuliman has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20801?usp=email )
Change subject: frame size at query level
......................................................................
frame size at query level
Change-Id: Icf52985cd22eed8ae93439d1faa187ce23d31630
---
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
M asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
8 files changed, 42 insertions(+), 22 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/01/20801/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
index ca97aa6..b6a9295 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
@@ -95,7 +95,7 @@
CompilerProperties.COMPILER_CBO_TEST_KEY,
CompilerProperties.COMPILER_FORCE_JOIN_ORDER_KEY,
CompilerProperties.COMPILER_QUERY_PLAN_SHAPE_KEY,
CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY,
CompilerProperties.COMPILER_COLUMN_FILTER_KEY,
CompilerProperties.COMPILER_BATCH_LOOKUP_KEY,
- FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
+ CompilerProperties.COMPILER_FRAMESIZE_KEY,
FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
CompilerProperties.COMPILER_MAX_VARIABLE_OCCURRENCES_INLINING_KEY,
CompilerProperties.COMPILER_DELTALAKE_FILESPLITS_KEY,
FuzzyUtils.SIM_FUNCTION_PROP_NAME,
FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
StartFeedStatement.WAIT_FOR_COMPLETION,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index 4b07a3b..319123b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -12866,7 +12866,7 @@
<test-case FilePath="tpch-sql-sugar">
<compilation-unit name="q17_large_gby_variant_parameter">
<output-dir compare="Text">none</output-dir>
- <expected-error>Invalid query parameter compiler.groupmemory -- value
has to be greater than or equal to</expected-error>
+ <expected-error>Invalid `compiler.groupmemory` "-10737418240" for
frame size=32768. value should be >= 4 * frame size: `compiler.groupmemory`
"131072" in bytes</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="tpch-sql-sugar">
@@ -12897,7 +12897,7 @@
<test-case FilePath="tpch-sql-sugar">
<compilation-unit name="q01_pricing_summary_report_parameter">
<output-dir compare="Text">none</output-dir>
- <expected-error>Invalid query parameter compiler.sortmemory -- value
has to be greater than or equal to</expected-error>
+ <expected-error>Invalid `compiler.sortmemory` "-7516192768" for frame
size=32768. value should be >= 3 * frame size: `compiler.sortmemory` "98304" in
bytes</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="tpch-sql-sugar">
@@ -12958,7 +12958,7 @@
<test-case FilePath="tpch-sql-sugar">
<compilation-unit name="q09_product_type_profit_parameter">
<output-dir compare="Text">none</output-dir>
- <expected-error>Invalid query parameter compiler.joinmemory -- value
has to be greater than or equal to</expected-error>
+ <expected-error>Invalid `compiler.joinmemory` "-34359738368" for frame
size=32768. value should be >= 5 * frame size: `compiler.joinmemory` "163840"
in bytes</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="tpch-sql-sugar">
@@ -16563,7 +16563,7 @@
<output-dir compare="Text">misc_01</output-dir>
<expected-error>ASX0002: Type mismatch</expected-error>
<expected-error>ASX1104: Invalid modifier FROM FIRST/LAST for
function</expected-error>
- <expected-error>ASX1037: Invalid query parameter
compiler.windowmemory</expected-error>
+ <expected-error>Invalid `compiler.windowmemory` "102400" for frame
size=32768. value should be >= 5 * frame size: `compiler.windowmemory` "163840"
in bytes </expected-error>
<expected-error>ASX1102: Expected window or aggregate function, got:
lowercase</expected-error>
<expected-error>ASX1079: Compilation error: count is a SQL-92
aggregate function</expected-error>
<expected-error>ASX1104: Invalid modifier RESPECT/IGNORE NULLS for
function</expected-error>
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index da5f39b..62be4d0 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -28,6 +28,7 @@
import static
org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
import static
org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
import static
org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER;
+import static
org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
import static
org.apache.hyracks.control.common.config.OptionTypes.getRangedIntegerType;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
@@ -67,7 +68,7 @@
StorageUtil.getIntSizeInBytes(8, KILOBYTE),
"The memory budget (in bytes) for an external scan operator
instance in a partition"),
COMPILER_FRAMESIZE(
- INTEGER_BYTE_UNIT,
+ POSITIVE_INTEGER_BYTE_UNIT,
StorageUtil.getIntSizeInBytes(32, KILOBYTE),
"The page size (in bytes) for computation"),
COMPILER_MIN_SORTMEMORY(
@@ -196,6 +197,8 @@
}
}
+ public static final String COMPILER_FRAMESIZE_KEY =
Option.COMPILER_FRAMESIZE.ini();
+
public static final String COMPILER_SORTMEMORY_KEY =
Option.COMPILER_SORTMEMORY.ini();
public static final String COMPILER_GROUPMEMORY_KEY =
Option.COMPILER_GROUPMEMORY.ini();
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
index 6de87a9..9ebf5c2 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java
@@ -51,8 +51,8 @@
public static PhysicalOptimizationConfig
createPhysicalOptimizationConf(CompilerProperties compilerProperties,
Map<String, Object> querySpecificConfig, Set<String>
parameterNames, SourceLocation sourceLoc)
throws AlgebricksException {
- int frameSize = compilerProperties.getFrameSize();
- int sortFrameLimit = getSortNumFrames(compilerProperties,
querySpecificConfig, sourceLoc);
+ int frameSize = getFrameSize(compilerProperties, querySpecificConfig,
sourceLoc);
+ int sortFrameLimit = getSortNumFrames(compilerProperties,
querySpecificConfig, sourceLoc, frameSize);
int groupFrameLimit =
getFrameLimit(CompilerProperties.COMPILER_GROUPMEMORY_KEY,
(String)
querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY),
compilerProperties.getGroupMemorySize(), frameSize,
MIN_FRAME_LIMIT_FOR_GROUP_BY, sourceLoc);
@@ -62,7 +62,8 @@
int windowFrameLimit =
getFrameLimit(CompilerProperties.COMPILER_WINDOWMEMORY_KEY,
(String)
querySpecificConfig.get(CompilerProperties.COMPILER_WINDOWMEMORY_KEY),
compilerProperties.getWindowMemorySize(), frameSize,
MIN_FRAME_LIMIT_FOR_WINDOW, sourceLoc);
- int textSearchFrameLimit = getTextSearchNumFrames(compilerProperties,
querySpecificConfig, sourceLoc);
+ int textSearchFrameLimit =
+ getTextSearchNumFrames(compilerProperties, frameSize,
querySpecificConfig, sourceLoc);
int sortNumSamples = getSortSamples(compilerProperties,
querySpecificConfig, sourceLoc);
boolean fullParallelSort = getBoolean(querySpecificConfig,
CompilerProperties.COMPILER_SORT_PARALLEL_KEY,
compilerProperties.getSortParallel());
@@ -143,6 +144,21 @@
return physOptConf;
}
+ private static int getFrameSize(CompilerProperties compilerProperties,
Map<String, Object> querySpecificConfig,
+ SourceLocation sourceLoc) throws AlgebricksException {
+ String queryFrameSize = (String)
querySpecificConfig.get(CompilerProperties.COMPILER_FRAMESIZE_KEY);
+ if (queryFrameSize == null) {
+ return compilerProperties.getFrameSize();
+ }
+ @SuppressWarnings("unchecked")
+ IOptionType<Integer> type =
CompilerProperties.Option.COMPILER_FRAMESIZE.type();
+ try {
+ return type.parse(queryFrameSize);
+ } catch (IllegalArgumentException e) {
+ throw AsterixException.create(ErrorCode.COMPILATION_ERROR, e,
sourceLoc, e.getMessage());
+ }
+ }
+
private static int getExternalScanBufferSize(String
externalScanMemorySizeParameter,
int compilerExternalScanMemorySize, SourceLocation sourceLoc)
throws AsterixException {
IOptionType<Integer> intByteParser = OptionTypes.INTEGER_BYTE_UNIT;
@@ -155,27 +171,25 @@
}
public static int getSortNumFrames(CompilerProperties compilerProperties,
Map<String, Object> querySpecificConfig,
- SourceLocation sourceLoc) throws AlgebricksException {
+ SourceLocation sourceLoc, int frameSize) throws
AlgebricksException {
return getFrameLimit(CompilerProperties.COMPILER_SORTMEMORY_KEY,
(String)
querySpecificConfig.get(CompilerProperties.COMPILER_SORTMEMORY_KEY),
- compilerProperties.getSortMemorySize(),
compilerProperties.getFrameSize(), MIN_FRAME_LIMIT_FOR_SORT,
- sourceLoc);
+ compilerProperties.getSortMemorySize(), frameSize,
MIN_FRAME_LIMIT_FOR_SORT, sourceLoc);
}
public static int getGroupByNumFrames(CompilerProperties
compilerProperties,
- Map<String, Object> querySpecificConfig, SourceLocation sourceLoc)
throws AlgebricksException {
+ Map<String, Object> querySpecificConfig, SourceLocation sourceLoc,
int frameSize)
+ throws AlgebricksException {
return getFrameLimit(CompilerProperties.COMPILER_GROUPMEMORY_KEY,
(String)
querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY),
- compilerProperties.getGroupMemorySize(),
compilerProperties.getFrameSize(),
- MIN_FRAME_LIMIT_FOR_GROUP_BY, sourceLoc);
+ compilerProperties.getGroupMemorySize(), frameSize,
MIN_FRAME_LIMIT_FOR_GROUP_BY, sourceLoc);
}
- public static int getTextSearchNumFrames(CompilerProperties
compilerProperties,
+ public static int getTextSearchNumFrames(CompilerProperties
compilerProperties, int frameSize,
Map<String, Object> querySpecificConfig, SourceLocation sourceLoc)
throws AlgebricksException {
return getFrameLimit(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY,
(String)
querySpecificConfig.get(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY),
- compilerProperties.getTextSearchMemorySize(),
compilerProperties.getFrameSize(),
- MIN_FRAME_LIMIT_FOR_TEXT_SEARCH, sourceLoc);
+ compilerProperties.getTextSearchMemorySize(), frameSize,
MIN_FRAME_LIMIT_FOR_TEXT_SEARCH, sourceLoc);
}
@SuppressWarnings("squid:S1166") // Either log or rethrow this exception
@@ -190,8 +204,8 @@
}
int frameLimit = (int) (memBudget / frameSize);
if (frameLimit < minFrameLimit) {
- throw
AsterixException.create(ErrorCode.COMPILATION_BAD_QUERY_PARAMETER_VALUE,
sourceLoc, parameterName,
- frameSize * minFrameLimit, "bytes");
+ throw
AsterixException.create(ErrorCode.INVALID_FRAME_BASED_MEMORY_BUDGET, sourceLoc,
parameterName,
+ memBudget, frameSize, minFrameLimit, frameSize *
minFrameLimit, "bytes");
}
// sets the frame limit to the minimum frame limit if the calculated
frame limit is too small.
return Math.max(frameLimit, minFrameLimit);
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index f604f66..d52fe07 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -346,6 +346,7 @@
ICEBERG_SNAPSHOT_ID_NOT_FOUND(1238),
INVALID_ICEBERG_SNAPSHOT_VALUE(1239),
NO_VALID_CREDENTIALS_PROVIDED_FOR_BIGLAKE_METASTORE_CATALOG(1240),
+ INVALID_FRAME_BASED_MEMORY_BUDGET(1241),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 16c5c99..4b0fd2b 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -348,6 +348,7 @@
1238 = Iceberg snapshot '%1$s' not found
1239 = Invalid Iceberg snapshot value: '%1$s'
1240 = No valid credentials provided to access Biglake Metastore catalog.
+1241 = Invalid `%1$s` "%2$s" for frame size=%3$s. value should be >= %4$s *
frame size: `%1$s` "%5$s" in %6$s
# Feed Errors
3001 = Illegal state.
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java
index 7e3af78..b14e0e5 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java
@@ -380,6 +380,6 @@
throws AlgebricksException {
return OptimizationConfUtil.getGroupByNumFrames(
metadataProvider.getApplicationContext().getCompilerProperties(),
metadataProvider.getConfig(),
- sourceLoc);
+ sourceLoc,
metadataProvider.getApplicationContext().getCompilerProperties().getFrameSize());
}
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index 8350141..b92dc28 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -158,7 +158,8 @@
private static int getSortNumFrames(MetadataProvider metadataProvider,
SourceLocation sourceLoc)
throws AlgebricksException {
return
OptimizationConfUtil.getSortNumFrames(metadataProvider.getApplicationContext().getCompilerProperties(),
- metadataProvider.getConfig(), sourceLoc);
+ metadataProvider.getConfig(), sourceLoc,
+
metadataProvider.getApplicationContext().getCompilerProperties().getFrameSize());
}
public static ISecondaryIndexOperationsHelper
createIndexOperationsHelper(Dataset dataset, Index index,
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20801?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Icf52985cd22eed8ae93439d1faa187ce23d31630
Gerrit-Change-Number: 20801
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <[email protected]>
