This is an automated email from the ASF dual-hosted git repository.
rongr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 44fe6940e1 [multistage] populate queryOption down to leaf (#10626)
44fe6940e1 is described below
commit 44fe6940e19cbf29d2f5f606124c2f6b77e85360
Author: Rong Rong <[email protected]>
AuthorDate: Mon Apr 17 20:15:13 2023 -0700
[multistage] populate queryOption down to leaf (#10626)
---
.../runtime/plan/ServerRequestPlanVisitor.java | 18 +++++++++++++---
.../pinot/query/runtime/QueryRunnerTest.java | 25 +++++++++++++++++-----
.../pinot/query/runtime/QueryRunnerTestBase.java | 12 ++++++++++-
3 files changed, 46 insertions(+), 9 deletions(-)
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/ServerRequestPlanVisitor.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/ServerRequestPlanVisitor.java
index 6255a99ddc..73dbc9acff 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/ServerRequestPlanVisitor.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/ServerRequestPlanVisitor.java
@@ -19,9 +19,9 @@
package org.apache.pinot.query.runtime.plan;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -126,8 +126,7 @@ public class ServerRequestPlanVisitor implements
StageNodeVisitor<Void, ServerPl
QUERY_OPTIMIZER.optimize(pinotQuery, tableConfig, schema);
// 2. set pinot query options according to requestMetadataMap
- pinotQuery.setQueryOptions(
-
ImmutableMap.of(CommonConstants.Broker.Request.QueryOptionKey.TIMEOUT_MS,
String.valueOf(timeoutMs)));
+ updateQueryOptions(pinotQuery, requestMetadataMap, timeoutMs,
traceEnabled);
// 3. wrapped around in broker request
BrokerRequest brokerRequest = new BrokerRequest();
@@ -151,6 +150,19 @@ public class ServerRequestPlanVisitor implements
StageNodeVisitor<Void, ServerPl
return context;
}
+ private static void updateQueryOptions(PinotQuery pinotQuery, Map<String,
String> requestMetadataMap, long timeoutMs,
+ boolean traceEnabled) {
+ Map<String, String> queryOptions = new HashMap<>();
+ // put default timeout and trace options
+ queryOptions.put(CommonConstants.Broker.Request.QueryOptionKey.TIMEOUT_MS,
String.valueOf(timeoutMs));
+ if (traceEnabled) {
+ queryOptions.put(CommonConstants.Broker.Request.TRACE, "true");
+ }
+ // overwrite with requestMetadataMap to carry query options from request:
+ queryOptions.putAll(requestMetadataMap);
+ pinotQuery.setQueryOptions(queryOptions);
+ }
+
private static void walkStageNode(StageNode node, ServerPlanRequestContext
context) {
node.visit(INSTANCE, context);
}
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
index e1e2e17ab1..1ebba8c439 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
@@ -54,10 +54,8 @@ import org.testng.annotations.Test;
/**
- * all legacy tests.
- *
- * @deprecated do not add to this test set. this class will be broken down and
clean up.
- * add your test to appropraite files in {@link
org.apache.pinot.query.runtime.queries} instead.
+ * all special tests that doesn't fit into {@link
org.apache.pinot.query.runtime.queries.ResourceBasedQueriesTest}
+ * pattern goes here.
*/
public class QueryRunnerTest extends QueryRunnerTestBase {
public static final Object[][] ROWS = new Object[][]{
@@ -156,13 +154,23 @@ public class QueryRunnerTest extends QueryRunnerTestBase {
_mailboxService.shutdown();
}
- @Test(dataProvider = "testDataWithSqlToFinalRowCount")
+
+ /**
+ * Test compares with expected row count only.
+ */
+ @Test(dataProvider = "testDataWithSqlToFinalRowCount")
public void testSqlWithFinalRowCountChecker(String sql, int expectedRows)
throws Exception {
List<Object[]> resultRows = queryRunner(sql, null);
Assert.assertEquals(resultRows.size(), expectedRows);
}
+ /**
+ * Test automatically compares against H2.
+ *
+ * @deprecated do not add to this test set. this class will be broken down
and clean up.
+ * add your test to the appropriate files in {@link
org.apache.pinot.query.runtime.queries} instead.
+ */
@Test(dataProvider = "testSql")
public void testSqlWithH2Checker(String sql)
throws Exception {
@@ -172,6 +180,9 @@ public class QueryRunnerTest extends QueryRunnerTestBase {
compareRowEquals(resultRows, expectedRows);
}
+ /**
+ * Test compares against its desired exceptions.
+ */
@Test(dataProvider = "testDataWithSqlExecutionExceptions")
public void testSqlWithExceptionMsgChecker(String sql, String exceptionMsg) {
long requestId = RANDOM_REQUEST_ID_GEN.nextLong();
@@ -235,6 +246,10 @@ public class QueryRunnerTest extends QueryRunnerTestBase {
new Object[]{"SELECT round_decimal(col3) FROM a", 15},
new Object[]{"SELECT col1, roundDecimal(COUNT(*)) FROM a GROUP BY
col1", 5},
new Object[]{"SELECT col1, round_decimal(COUNT(*)) FROM a GROUP BY
col1", 5},
+
+ // test queries with special query options attached
+ // - when leaf limit is set, each server returns multiStageLeafLimit
number of rows only.
+ new Object[]{"SET multiStageLeafLimit = 1; SELECT * FROM a", 2},
};
}
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTestBase.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTestBase.java
index affc0317a0..d93a385e62 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTestBase.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTestBase.java
@@ -61,6 +61,8 @@ import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.StringUtil;
+import org.apache.pinot.sql.parsers.CalciteSqlParser;
+import org.apache.pinot.sql.parsers.SqlNodeAndOptions;
import org.h2.jdbc.JdbcArray;
import org.testng.Assert;
@@ -84,14 +86,22 @@ public abstract class QueryRunnerTestBase extends
QueryTestSet {
// --------------------------------------------------------------------------
// QUERY UTILS
// --------------------------------------------------------------------------
+
+ /**
+ * Dispatch query to each pinot-server. The logic should mimic
+ * {@link QueryDispatcher#submit(long, QueryPlan, long, Map)} but does not
actually make ser/de dispatches.
+ */
protected List<Object[]> queryRunner(String sql, Map<Integer,
ExecutionStatsAggregator> executionStatsAggregatorMap) {
- QueryPlan queryPlan = _queryEnvironment.planQuery(sql);
long requestId = RANDOM_REQUEST_ID_GEN.nextLong();
+ SqlNodeAndOptions sqlNodeAndOptions =
CalciteSqlParser.compileToSqlNodeAndOptions(sql);
+ QueryPlan queryPlan = _queryEnvironment.planQuery(sql, sqlNodeAndOptions,
requestId).getQueryPlan();
Map<String, String> requestMetadataMap = new HashMap<>();
requestMetadataMap.put(QueryConfig.KEY_OF_BROKER_REQUEST_ID,
String.valueOf(requestId));
requestMetadataMap.put(QueryConfig.KEY_OF_BROKER_REQUEST_TIMEOUT_MS,
String.valueOf(CommonConstants.Broker.DEFAULT_BROKER_TIMEOUT_MS));
+ requestMetadataMap.putAll(sqlNodeAndOptions.getOptions());
+ // Putting trace testing here as extra options as it doesn't go along with
the rest of the items.
if (executionStatsAggregatorMap != null) {
requestMetadataMap.put(CommonConstants.Broker.Request.TRACE, "true");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]