This is an automated email from the ASF dual-hosted git repository.
ankitsultana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a6a3b402e7 [timeseries] Part-4: Complete Support for Multi-Server
Queries (#14676)
a6a3b402e7 is described below
commit a6a3b402e795a6a68c253a4fe4d14c56a3e4acc8
Author: Ankit Sultana <[email protected]>
AuthorDate: Tue Dec 24 22:06:34 2024 -0600
[timeseries] Part-4: Complete Support for Multi-Server Queries (#14676)
---
.../requesthandler/TimeSeriesRequestHandler.java | 6 ++
.../apache/pinot/query/runtime/QueryRunner.java | 73 ++++++++-----
.../PhysicalTimeSeriesBrokerPlanVisitor.java | 79 ++++++++++++++
.../PhysicalTimeSeriesServerPlanVisitor.java | 27 +++--
.../timeseries/TimeSeriesExecutionContext.java | 30 +++++-
.../query/service/dispatch/QueryDispatcher.java | 120 +++++++++++----------
.../timeseries/TimeSeriesDispatchClient.java | 10 +-
.../timeseries/TimeSeriesDispatchObserver.java | 67 ++++++++----
.../pinot/query/service/server/QueryServer.java | 3 +-
.../PhysicalTimeSeriesServerPlanVisitorTest.java | 21 ++--
.../tsdb/planner/TimeSeriesQueryEnvironment.java | 86 ++++++---------
.../tsdb/planner/physical/TableScanVisitor.java | 33 ++++--
.../physical/TimeSeriesDispatchablePlan.java | 54 +++++++---
.../series/TimeSeriesBuilderFactoryProvider.java | 2 +-
.../pinot/tools/TimeSeriesEngineQuickStart.java | 2 +-
15 files changed, 404 insertions(+), 209 deletions(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/TimeSeriesRequestHandler.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/TimeSeriesRequestHandler.java
index 52cf63f562..d14f286013 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/TimeSeriesRequestHandler.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/TimeSeriesRequestHandler.java
@@ -53,6 +53,7 @@ import
org.apache.pinot.tsdb.planner.TimeSeriesQueryEnvironment;
import org.apache.pinot.tsdb.planner.physical.TimeSeriesDispatchablePlan;
import org.apache.pinot.tsdb.spi.RangeTimeSeriesRequest;
import org.apache.pinot.tsdb.spi.TimeSeriesLogicalPlanResult;
+import org.apache.pinot.tsdb.spi.series.TimeSeriesBuilderFactoryProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,6 +71,7 @@ public class TimeSeriesRequestHandler extends
BaseBrokerRequestHandler {
_queryEnvironment = new TimeSeriesQueryEnvironment(config, routingManager,
tableCache);
_queryEnvironment.init(config);
_queryDispatcher = queryDispatcher;
+ TimeSeriesBuilderFactoryProvider.init(config);
}
@Override
@@ -117,6 +119,10 @@ public class TimeSeriesRequestHandler extends
BaseBrokerRequestHandler {
if (timeSeriesResponse == null
||
timeSeriesResponse.getStatus().equals(PinotBrokerTimeSeriesResponse.ERROR_STATUS))
{
_brokerMetrics.addMeteredGlobalValue(BrokerMeter.TIME_SERIES_GLOBAL_QUERIES_FAILED,
1);
+ final String errorMessage = timeSeriesResponse == null ? "null
time-series response"
+ : timeSeriesResponse.getError();
+ // TODO(timeseries): Remove logging for failed queries.
+ LOGGER.warn("time-series query failed with error: {}", errorMessage);
}
}
}
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java
index 0ca99b06cc..ac335a1674 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java
@@ -20,11 +20,11 @@ package org.apache.pinot.query.runtime;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.protobuf.ByteString;
+import com.google.common.collect.ImmutableMap;
import io.grpc.stub.StreamObserver;
-import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -36,12 +36,12 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.helix.HelixManager;
import org.apache.pinot.common.config.TlsConfig;
import org.apache.pinot.common.datatable.StatMap;
import org.apache.pinot.common.metrics.ServerMetrics;
import org.apache.pinot.common.proto.Worker;
-import org.apache.pinot.common.response.PinotBrokerTimeSeriesResponse;
import org.apache.pinot.common.utils.config.QueryOptionsUtils;
import org.apache.pinot.core.data.manager.InstanceDataManager;
import org.apache.pinot.core.query.executor.QueryExecutor;
@@ -69,6 +69,7 @@ import
org.apache.pinot.query.runtime.plan.pipeline.PipelineBreakerResult;
import org.apache.pinot.query.runtime.plan.server.ServerPlanRequestUtils;
import
org.apache.pinot.query.runtime.timeseries.PhysicalTimeSeriesServerPlanVisitor;
import org.apache.pinot.query.runtime.timeseries.TimeSeriesExecutionContext;
+import org.apache.pinot.query.runtime.timeseries.serde.TimeSeriesBlockSerde;
import org.apache.pinot.spi.accounting.ThreadExecutionContext;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.executor.ExecutorServiceUtils;
@@ -258,45 +259,65 @@ public class QueryRunner {
* TODO: This design is at odds with MSE because MSE runs even the leaf
stage via OpChainSchedulerService.
* However, both OpChain scheduler and this method use the same
ExecutorService.
*/
- public void processTimeSeriesQuery(String serializedPlan, Map<String,
String> metadata,
+ public void processTimeSeriesQuery(List<String> serializedPlanFragments,
Map<String, String> metadata,
StreamObserver<Worker.TimeSeriesResponse> responseObserver) {
// Define a common way to handle errors.
- final Consumer<Throwable> handleErrors = (t) -> {
- Map<String, String> errorMetadata = new HashMap<>();
- errorMetadata.put(WorkerResponseMetadataKeys.ERROR_TYPE,
t.getClass().getSimpleName());
- errorMetadata.put(WorkerResponseMetadataKeys.ERROR_MESSAGE,
t.getMessage() == null
- ? "Unknown error: no message" : t.getMessage());
-
responseObserver.onNext(Worker.TimeSeriesResponse.newBuilder().putAllMetadata(errorMetadata).build());
- responseObserver.onCompleted();
+ final Consumer<Pair<Throwable, String>> handleErrors = (pair) -> {
+ Throwable t = pair.getLeft();
+ try {
+ String planId = pair.getRight();
+ Map<String, String> errorMetadata = new HashMap<>();
+ errorMetadata.put(WorkerResponseMetadataKeys.ERROR_TYPE,
t.getClass().getSimpleName());
+ errorMetadata.put(WorkerResponseMetadataKeys.ERROR_MESSAGE,
t.getMessage() == null
+ ? "Unknown error: no message" : t.getMessage());
+ errorMetadata.put(WorkerResponseMetadataKeys.PLAN_ID, planId);
+ // TODO(timeseries): remove logging for failed queries.
+ LOGGER.warn("time-series query failed:", t);
+
responseObserver.onNext(Worker.TimeSeriesResponse.newBuilder().putAllMetadata(errorMetadata).build());
+ responseObserver.onCompleted();
+ } catch (Throwable t2) {
+ LOGGER.warn("Unable to send error to broker. Original error: {}",
t.getMessage(), t2);
+ }
};
+ if (serializedPlanFragments.isEmpty()) {
+ handleErrors.accept(Pair.of(new IllegalStateException("No plan fragments
received in server"), ""));
+ return;
+ }
try {
final long deadlineMs = extractDeadlineMs(metadata);
Preconditions.checkState(System.currentTimeMillis() < deadlineMs,
- "Query timed out before getting processed in server. Remaining time:
%s", deadlineMs);
- // Deserialize plan, and compile to create a tree of operators.
- BaseTimeSeriesPlanNode rootNode =
TimeSeriesPlanSerde.deserialize(serializedPlan);
+ "Query timed out before getting processed in server. Exceeded time
by (ms): %s",
+ System.currentTimeMillis() - deadlineMs);
+ List<BaseTimeSeriesPlanNode> fragmentRoots =
serializedPlanFragments.stream()
+ .map(TimeSeriesPlanSerde::deserialize).collect(Collectors.toList());
TimeSeriesExecutionContext context = new TimeSeriesExecutionContext(
- metadata.get(WorkerRequestMetadataKeys.LANGUAGE),
extractTimeBuckets(metadata),
- extractPlanToSegmentMap(metadata), deadlineMs, metadata);
- BaseTimeSeriesOperator operator =
_timeSeriesPhysicalPlanVisitor.compile(rootNode, context);
+ metadata.get(WorkerRequestMetadataKeys.LANGUAGE),
extractTimeBuckets(metadata), deadlineMs, metadata,
+ extractPlanToSegmentMap(metadata), Collections.emptyMap());
+ final List<BaseTimeSeriesOperator> fragmentOpChains =
fragmentRoots.stream().map(x -> {
+ return _timeSeriesPhysicalPlanVisitor.compile(x, context);
+ }).collect(Collectors.toList());
// Run the operator using the same executor service as
OpChainSchedulerService
_executorService.submit(() -> {
+ String currentPlanId = "";
try {
- TimeSeriesBlock seriesBlock = operator.nextBlock();
- Worker.TimeSeriesResponse response =
Worker.TimeSeriesResponse.newBuilder()
- .setPayload(ByteString.copyFrom(
-
PinotBrokerTimeSeriesResponse.fromTimeSeriesBlock(seriesBlock).serialize(),
- StandardCharsets.UTF_8))
- .build();
- responseObserver.onNext(response);
+ for (int index = 0; index < fragmentOpChains.size(); index++) {
+ currentPlanId = fragmentRoots.get(index).getId();
+ BaseTimeSeriesOperator fragmentOpChain =
fragmentOpChains.get(index);
+ TimeSeriesBlock seriesBlock = fragmentOpChain.nextBlock();
+ Worker.TimeSeriesResponse response =
Worker.TimeSeriesResponse.newBuilder()
+
.setPayload(TimeSeriesBlockSerde.serializeTimeSeriesBlock(seriesBlock))
+
.putAllMetadata(ImmutableMap.of(WorkerResponseMetadataKeys.PLAN_ID,
currentPlanId))
+ .build();
+ responseObserver.onNext(response);
+ }
responseObserver.onCompleted();
} catch (Throwable t) {
- handleErrors.accept(t);
+ handleErrors.accept(Pair.of(t, currentPlanId));
}
});
} catch (Throwable t) {
LOGGER.error("Error running time-series query", t);
- handleErrors.accept(t);
+ handleErrors.accept(Pair.of(t, ""));
}
}
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesBrokerPlanVisitor.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesBrokerPlanVisitor.java
new file mode 100644
index 0000000000..533c4e1bb1
--- /dev/null
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesBrokerPlanVisitor.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.query.runtime.timeseries;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.BlockingQueue;
+import org.apache.pinot.tsdb.planner.TimeSeriesExchangeNode;
+import org.apache.pinot.tsdb.spi.operator.BaseTimeSeriesOperator;
+import org.apache.pinot.tsdb.spi.plan.BaseTimeSeriesPlanNode;
+import org.apache.pinot.tsdb.spi.plan.LeafTimeSeriesPlanNode;
+
+
+public class PhysicalTimeSeriesBrokerPlanVisitor {
+ // Warning: Don't use singleton access pattern, since Quickstarts run in a
single JVM and spawn multiple broker/server
+ public PhysicalTimeSeriesBrokerPlanVisitor() {
+ }
+
+ public void init() {
+ }
+
+ public BaseTimeSeriesOperator compile(BaseTimeSeriesPlanNode rootNode,
TimeSeriesExecutionContext context,
+ Map<String, Integer> numInputServersByExchangeNode) {
+ // Step-1: Replace time series exchange node with its Physical Plan Node.
+ rootNode = initExchangeReceivePlanNode(rootNode, context,
numInputServersByExchangeNode);
+ // Step-2: Trigger recursive operator generation
+ return rootNode.run();
+ }
+
+ public BaseTimeSeriesPlanNode
initExchangeReceivePlanNode(BaseTimeSeriesPlanNode planNode,
+ TimeSeriesExecutionContext context, Map<String, Integer>
numInputServersByExchangeNode) {
+ if (planNode instanceof LeafTimeSeriesPlanNode) {
+ throw new IllegalStateException("Found leaf time series plan node in
broker");
+ } else if (planNode instanceof TimeSeriesExchangeNode) {
+ int numInputServers =
numInputServersByExchangeNode.get(planNode.getId());
+ return compileToPhysicalReceiveNode((TimeSeriesExchangeNode) planNode,
context, numInputServers);
+ }
+ List<BaseTimeSeriesPlanNode> newInputs = new ArrayList<>();
+ for (int index = 0; index < planNode.getInputs().size(); index++) {
+ BaseTimeSeriesPlanNode inputNode = planNode.getInputs().get(index);
+ if (inputNode instanceof TimeSeriesExchangeNode) {
+ int numInputServers =
numInputServersByExchangeNode.get(inputNode.getId());
+ TimeSeriesExchangeReceivePlanNode exchangeReceivePlanNode =
compileToPhysicalReceiveNode(
+ (TimeSeriesExchangeNode) inputNode, context, numInputServers);
+ newInputs.add(exchangeReceivePlanNode);
+ } else {
+ newInputs.add(initExchangeReceivePlanNode(inputNode, context,
numInputServersByExchangeNode));
+ }
+ }
+ return planNode.withInputs(newInputs);
+ }
+
+ TimeSeriesExchangeReceivePlanNode
compileToPhysicalReceiveNode(TimeSeriesExchangeNode exchangeNode,
+ TimeSeriesExecutionContext context, int numServersQueried) {
+ TimeSeriesExchangeReceivePlanNode exchangeReceivePlanNode = new
TimeSeriesExchangeReceivePlanNode(
+ exchangeNode.getId(), context.getDeadlineMs(),
exchangeNode.getAggInfo(), context.getSeriesBuilderFactory());
+ BlockingQueue<Object> receiver =
context.getExchangeReceiverByPlanId().get(exchangeNode.getId());
+ exchangeReceivePlanNode.init(Objects.requireNonNull(receiver, "No receiver
for node"), numServersQueried);
+ return exchangeReceivePlanNode;
+ }
+}
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesServerPlanVisitor.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesServerPlanVisitor.java
index 9ed93d11a0..b2be6b2f56 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesServerPlanVisitor.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesServerPlanVisitor.java
@@ -20,6 +20,7 @@ package org.apache.pinot.query.runtime.timeseries;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -56,26 +57,34 @@ public class PhysicalTimeSeriesServerPlanVisitor {
}
public BaseTimeSeriesOperator compile(BaseTimeSeriesPlanNode rootNode,
TimeSeriesExecutionContext context) {
- // Step-1: Replace scan filter project with our physical plan node with
Pinot Core and Runtime context
- initLeafPlanNode(rootNode, context);
+ // Step-1: Replace leaf node with our physical plan node with Pinot Core
and Runtime context
+ rootNode = initLeafPlanNode(rootNode, context);
// Step-2: Trigger recursive operator generation
return rootNode.run();
}
- public void initLeafPlanNode(BaseTimeSeriesPlanNode planNode,
TimeSeriesExecutionContext context) {
+ public BaseTimeSeriesPlanNode initLeafPlanNode(BaseTimeSeriesPlanNode
planNode, TimeSeriesExecutionContext context) {
+ if (planNode instanceof LeafTimeSeriesPlanNode) {
+ return convertLeafToPhysicalTableScan((LeafTimeSeriesPlanNode) planNode,
context);
+ }
+ List<BaseTimeSeriesPlanNode> newInputs = new ArrayList<>();
for (int index = 0; index < planNode.getInputs().size(); index++) {
BaseTimeSeriesPlanNode childNode = planNode.getInputs().get(index);
if (childNode instanceof LeafTimeSeriesPlanNode) {
LeafTimeSeriesPlanNode leafNode = (LeafTimeSeriesPlanNode) childNode;
- List<String> segments =
context.getPlanIdToSegmentsMap().get(leafNode.getId());
- ServerQueryRequest serverQueryRequest =
compileLeafServerQueryRequest(leafNode, segments, context);
- TimeSeriesPhysicalTableScan physicalTableScan = new
TimeSeriesPhysicalTableScan(childNode.getId(),
- serverQueryRequest, _queryExecutor, _executorService);
- planNode.getInputs().set(index, physicalTableScan);
+ newInputs.add(convertLeafToPhysicalTableScan(leafNode, context));
} else {
- initLeafPlanNode(childNode, context);
+ newInputs.add(initLeafPlanNode(childNode, context));
}
}
+ return planNode.withInputs(newInputs);
+ }
+
+ private TimeSeriesPhysicalTableScan
convertLeafToPhysicalTableScan(LeafTimeSeriesPlanNode leafNode,
+ TimeSeriesExecutionContext context) {
+ List<String> segments =
context.getPlanIdToSegmentsMap().getOrDefault(leafNode.getId(),
Collections.emptyList());
+ ServerQueryRequest serverQueryRequest =
compileLeafServerQueryRequest(leafNode, segments, context);
+ return new TimeSeriesPhysicalTableScan(leafNode.getId(),
serverQueryRequest, _queryExecutor, _executorService);
}
public ServerQueryRequest
compileLeafServerQueryRequest(LeafTimeSeriesPlanNode leafNode, List<String>
segments,
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/TimeSeriesExecutionContext.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/TimeSeriesExecutionContext.java
index e8469ff495..74f62329e9 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/TimeSeriesExecutionContext.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/timeseries/TimeSeriesExecutionContext.java
@@ -20,23 +20,31 @@ package org.apache.pinot.query.runtime.timeseries;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.BlockingQueue;
import org.apache.pinot.tsdb.spi.TimeBuckets;
+import org.apache.pinot.tsdb.spi.series.TimeSeriesBuilderFactory;
+import org.apache.pinot.tsdb.spi.series.TimeSeriesBuilderFactoryProvider;
public class TimeSeriesExecutionContext {
private final String _language;
private final TimeBuckets _initialTimeBuckets;
private final Map<String, List<String>> _planIdToSegmentsMap;
+ private final Map<String, BlockingQueue<Object>> _exchangeReceiverByPlanId;
private final long _deadlineMs;
private final Map<String, String> _metadataMap;
+ private final TimeSeriesBuilderFactory _seriesBuilderFactory;
- public TimeSeriesExecutionContext(String language, TimeBuckets
initialTimeBuckets,
- Map<String, List<String>> planIdToSegmentsMap, long deadlineMs,
Map<String, String> metadataMap) {
+ public TimeSeriesExecutionContext(String language, TimeBuckets
initialTimeBuckets, long deadlineMs,
+ Map<String, String> metadataMap, Map<String, List<String>>
planIdToSegmentsMap,
+ Map<String, BlockingQueue<Object>> exchangeReceiverByPlanId) {
_language = language;
_initialTimeBuckets = initialTimeBuckets;
- _planIdToSegmentsMap = planIdToSegmentsMap;
_deadlineMs = deadlineMs;
_metadataMap = metadataMap;
+ _planIdToSegmentsMap = planIdToSegmentsMap;
+ _exchangeReceiverByPlanId = exchangeReceiverByPlanId;
+ _seriesBuilderFactory =
TimeSeriesBuilderFactoryProvider.getSeriesBuilderFactory(language);
}
public String getLanguage() {
@@ -47,8 +55,8 @@ public class TimeSeriesExecutionContext {
return _initialTimeBuckets;
}
- public Map<String, List<String>> getPlanIdToSegmentsMap() {
- return _planIdToSegmentsMap;
+ public long getDeadlineMs() {
+ return _deadlineMs;
}
public long getRemainingTimeMs() {
@@ -58,4 +66,16 @@ public class TimeSeriesExecutionContext {
public Map<String, String> getMetadataMap() {
return _metadataMap;
}
+
+ public Map<String, List<String>> getPlanIdToSegmentsMap() {
+ return _planIdToSegmentsMap;
+ }
+
+ public Map<String, BlockingQueue<Object>> getExchangeReceiverByPlanId() {
+ return _exchangeReceiverByPlanId;
+ }
+
+ public TimeSeriesBuilderFactory getSeriesBuilderFactory() {
+ return _seriesBuilderFactory;
+ }
}
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/QueryDispatcher.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/QueryDispatcher.java
index b791f1ec58..5a4ce98286 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/QueryDispatcher.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/QueryDispatcher.java
@@ -71,17 +71,22 @@ import
org.apache.pinot.query.runtime.blocks.TransferableBlockUtils;
import org.apache.pinot.query.runtime.operator.MailboxReceiveOperator;
import org.apache.pinot.query.runtime.plan.MultiStageQueryStats;
import org.apache.pinot.query.runtime.plan.OpChainExecutionContext;
-import
org.apache.pinot.query.service.dispatch.timeseries.AsyncQueryTimeSeriesDispatchResponse;
+import
org.apache.pinot.query.runtime.timeseries.PhysicalTimeSeriesBrokerPlanVisitor;
+import org.apache.pinot.query.runtime.timeseries.TimeSeriesExecutionContext;
import
org.apache.pinot.query.service.dispatch.timeseries.TimeSeriesDispatchClient;
+import
org.apache.pinot.query.service.dispatch.timeseries.TimeSeriesDispatchObserver;
import org.apache.pinot.spi.accounting.ThreadExecutionContext;
import org.apache.pinot.spi.trace.RequestContext;
import org.apache.pinot.spi.trace.Tracing;
import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.tsdb.planner.TimeSeriesExchangeNode;
import
org.apache.pinot.tsdb.planner.TimeSeriesPlanConstants.WorkerRequestMetadataKeys;
-import
org.apache.pinot.tsdb.planner.TimeSeriesPlanConstants.WorkerResponseMetadataKeys;
import org.apache.pinot.tsdb.planner.physical.TimeSeriesDispatchablePlan;
import org.apache.pinot.tsdb.planner.physical.TimeSeriesQueryServerInstance;
import org.apache.pinot.tsdb.spi.TimeBuckets;
+import org.apache.pinot.tsdb.spi.operator.BaseTimeSeriesOperator;
+import org.apache.pinot.tsdb.spi.plan.BaseTimeSeriesPlanNode;
+import org.apache.pinot.tsdb.spi.series.TimeSeriesBlock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,6 +105,8 @@ public class QueryDispatcher {
private final Map<String, TimeSeriesDispatchClient>
_timeSeriesDispatchClientMap = new ConcurrentHashMap<>();
@Nullable
private final TlsConfig _tlsConfig;
+ private final PhysicalTimeSeriesBrokerPlanVisitor
_timeSeriesBrokerPlanVisitor
+ = new PhysicalTimeSeriesBrokerPlanVisitor();
public QueryDispatcher(MailboxService mailboxService) {
this(mailboxService, null);
@@ -169,41 +176,6 @@ public class QueryDispatcher {
return planNodes;
}
- public PinotBrokerTimeSeriesResponse submitAndGet(RequestContext context,
TimeSeriesDispatchablePlan plan,
- long timeoutMs, Map<String, String> queryOptions) {
- long requestId = context.getRequestId();
- BlockingQueue<AsyncQueryTimeSeriesDispatchResponse> receiver = new
ArrayBlockingQueue<>(10);
- try {
- submit(requestId, plan, timeoutMs, queryOptions, context,
receiver::offer);
- AsyncQueryTimeSeriesDispatchResponse received = receiver.poll(timeoutMs,
TimeUnit.MILLISECONDS);
- if (received == null) {
- return PinotBrokerTimeSeriesResponse.newErrorResponse(
- "TimeoutException", "Timed out waiting for response");
- }
- if (received.getThrowable() != null) {
- Throwable t = received.getThrowable();
- return
PinotBrokerTimeSeriesResponse.newErrorResponse(t.getClass().getSimpleName(),
t.getMessage());
- }
- if (received.getQueryResponse() == null) {
- return PinotBrokerTimeSeriesResponse.newErrorResponse("NullResponse",
"Received null response from server");
- }
- if (received.getQueryResponse().containsMetadata(
- WorkerResponseMetadataKeys.ERROR_MESSAGE)) {
- return PinotBrokerTimeSeriesResponse.newErrorResponse(
- received.getQueryResponse().getMetadataOrDefault(
- WorkerResponseMetadataKeys.ERROR_TYPE, "unknown error-type"),
- received.getQueryResponse().getMetadataOrDefault(
- WorkerResponseMetadataKeys.ERROR_MESSAGE, "unknown error"));
- }
- Worker.TimeSeriesResponse timeSeriesResponse =
received.getQueryResponse();
- Preconditions.checkNotNull(timeSeriesResponse, "time series response is
null");
- return OBJECT_MAPPER.readValue(
- timeSeriesResponse.getPayload().toStringUtf8(),
PinotBrokerTimeSeriesResponse.class);
- } catch (Throwable t) {
- return
PinotBrokerTimeSeriesResponse.newErrorResponse(t.getClass().getSimpleName(),
t.getMessage());
- }
- }
-
@VisibleForTesting
void submit(long requestId, DispatchableSubPlan dispatchableSubPlan, long
timeoutMs, Map<String, String> queryOptions)
throws Exception {
@@ -283,25 +255,8 @@ public class QueryDispatcher {
}
}
- void submit(long requestId, TimeSeriesDispatchablePlan plan, long timeoutMs,
Map<String, String> queryOptions,
- RequestContext requestContext,
Consumer<AsyncQueryTimeSeriesDispatchResponse> receiver)
- throws Exception {
- Deadline deadline = Deadline.after(timeoutMs, TimeUnit.MILLISECONDS);
- long deadlineMs = System.currentTimeMillis() + timeoutMs;
- String serializedPlan = plan.getSerializedPlan();
- Worker.TimeSeriesQueryRequest request =
Worker.TimeSeriesQueryRequest.newBuilder()
- .addDispatchPlan(serializedPlan)
- .putAllMetadata(initializeTimeSeriesMetadataMap(plan, deadlineMs,
requestContext))
- .putMetadata(CommonConstants.Query.Request.MetadataKeys.REQUEST_ID,
Long.toString(requestId))
- .build();
-
getOrCreateTimeSeriesDispatchClient(plan.getQueryServerInstance()).submit(request,
- new QueryServerInstance(plan.getQueryServerInstance().getHostname(),
- plan.getQueryServerInstance().getQueryServicePort(),
plan.getQueryServerInstance().getQueryMailboxPort()),
- deadline, receiver::accept);
- };
-
Map<String, String>
initializeTimeSeriesMetadataMap(TimeSeriesDispatchablePlan dispatchablePlan,
long deadlineMs,
- RequestContext requestContext) {
+ RequestContext requestContext, String instanceId) {
Map<String, String> result = new HashMap<>();
TimeBuckets timeBuckets = dispatchablePlan.getTimeBuckets();
result.put(WorkerRequestMetadataKeys.LANGUAGE,
dispatchablePlan.getLanguage());
@@ -309,7 +264,8 @@ public class QueryDispatcher {
result.put(WorkerRequestMetadataKeys.WINDOW_SECONDS,
Long.toString(timeBuckets.getBucketSize().getSeconds()));
result.put(WorkerRequestMetadataKeys.NUM_ELEMENTS,
Long.toString(timeBuckets.getTimeBuckets().length));
result.put(WorkerRequestMetadataKeys.DEADLINE_MS,
Long.toString(deadlineMs));
- for (Map.Entry<String, List<String>> entry :
dispatchablePlan.getPlanIdToSegments().entrySet()) {
+ Map<String, List<String>> leafIdToSegments =
dispatchablePlan.getLeafIdToSegmentsByInstanceId().get(instanceId);
+ for (Map.Entry<String, List<String>> entry : leafIdToSegments.entrySet()) {
result.put(WorkerRequestMetadataKeys.encodeSegmentListKey(entry.getKey()),
String.join(",", entry.getValue()));
}
result.put(CommonConstants.Query.Request.MetadataKeys.REQUEST_ID,
Long.toString(requestContext.getRequestId()));
@@ -513,6 +469,58 @@ public class QueryDispatcher {
_executorService.shutdown();
}
+ public PinotBrokerTimeSeriesResponse submitAndGet(RequestContext context,
TimeSeriesDispatchablePlan plan,
+ long timeoutMs, Map<String, String> queryOptions) {
+ long requestId = context.getRequestId();
+ try {
+ TimeSeriesBlock result = submitAndGet(requestId, plan, timeoutMs,
queryOptions, context);
+ return PinotBrokerTimeSeriesResponse.fromTimeSeriesBlock(result);
+ } catch (Throwable t) {
+ return
PinotBrokerTimeSeriesResponse.newErrorResponse(t.getClass().getSimpleName(),
t.getMessage());
+ }
+ }
+
+ TimeSeriesBlock submitAndGet(long requestId, TimeSeriesDispatchablePlan
plan, long timeoutMs,
+ Map<String, String> queryOptions, RequestContext requestContext)
+ throws Exception {
+ long deadlineMs = System.currentTimeMillis() + timeoutMs;
+ BaseTimeSeriesPlanNode brokerFragment = plan.getBrokerFragment();
+ // Get consumers for leafs
+ Map<String, BlockingQueue<Object>> receiversByPlanId = new HashMap<>();
+ populateConsumers(brokerFragment, receiversByPlanId);
+ // Compile brokerFragment to get operators
+ TimeSeriesExecutionContext brokerExecutionContext = new
TimeSeriesExecutionContext(plan.getLanguage(),
+ plan.getTimeBuckets(), deadlineMs, Collections.emptyMap(),
Collections.emptyMap(), receiversByPlanId);
+ BaseTimeSeriesOperator brokerOperator =
_timeSeriesBrokerPlanVisitor.compile(brokerFragment,
+ brokerExecutionContext, plan.getNumInputServersForExchangePlanNode());
+ // Create dispatch observer for each query server
+ for (TimeSeriesQueryServerInstance serverInstance :
plan.getQueryServerInstances()) {
+ String serverId = serverInstance.getInstanceId();
+ Deadline deadline = Deadline.after(deadlineMs -
System.currentTimeMillis(), TimeUnit.MILLISECONDS);
+ Preconditions.checkState(!deadline.isExpired(), "Deadline expired before
query could be sent to servers");
+ // Send server fragment to every server
+ Worker.TimeSeriesQueryRequest request =
Worker.TimeSeriesQueryRequest.newBuilder()
+ .addAllDispatchPlan(plan.getSerializedServerFragments())
+ .putAllMetadata(initializeTimeSeriesMetadataMap(plan, deadlineMs,
requestContext, serverId))
+ .putMetadata(CommonConstants.Query.Request.MetadataKeys.REQUEST_ID,
Long.toString(requestId))
+ .build();
+ TimeSeriesDispatchObserver
+ dispatchObserver = new TimeSeriesDispatchObserver(receiversByPlanId);
+ getOrCreateTimeSeriesDispatchClient(serverInstance).submit(request,
deadline, dispatchObserver);
+ }
+ // Execute broker fragment
+ return brokerOperator.nextBlock();
+ }
+
+ private void populateConsumers(BaseTimeSeriesPlanNode planNode, Map<String,
BlockingQueue<Object>> receiverMap) {
+ if (planNode instanceof TimeSeriesExchangeNode) {
+ receiverMap.put(planNode.getId(), new
ArrayBlockingQueue<>(TimeSeriesDispatchObserver.MAX_QUEUE_CAPACITY));
+ }
+ for (BaseTimeSeriesPlanNode childNode : planNode.getInputs()) {
+ populateConsumers(childNode, receiverMap);
+ }
+ }
+
public static class QueryResult {
private final ResultTable _resultTable;
private final List<MultiStageQueryStats.StageStats.Closed> _queryStats;
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/timeseries/TimeSeriesDispatchClient.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/timeseries/TimeSeriesDispatchClient.java
index df77344665..6dc6bc3141 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/timeseries/TimeSeriesDispatchClient.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/timeseries/TimeSeriesDispatchClient.java
@@ -21,10 +21,9 @@ package org.apache.pinot.query.service.dispatch.timeseries;
import io.grpc.Deadline;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
-import java.util.function.Consumer;
+import io.grpc.stub.StreamObserver;
import org.apache.pinot.common.proto.PinotQueryWorkerGrpc;
import org.apache.pinot.common.proto.Worker;
-import org.apache.pinot.query.routing.QueryServerInstance;
/**
@@ -48,9 +47,8 @@ public class TimeSeriesDispatchClient {
return _channel;
}
- public void submit(Worker.TimeSeriesQueryRequest request,
QueryServerInstance virtualServer, Deadline deadline,
- Consumer<AsyncQueryTimeSeriesDispatchResponse> callback) {
- _dispatchStub.withDeadline(deadline).submitTimeSeries(
- request, new TimeSeriesDispatchObserver(virtualServer, callback));
+ public void submit(Worker.TimeSeriesQueryRequest request, Deadline deadline,
+ StreamObserver<Worker.TimeSeriesResponse> responseStreamObserver) {
+ _dispatchStub.withDeadline(deadline).submitTimeSeries(request,
responseStreamObserver);
}
}
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/timeseries/TimeSeriesDispatchObserver.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/timeseries/TimeSeriesDispatchObserver.java
index ccfe0e122c..599ce414c0 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/timeseries/TimeSeriesDispatchObserver.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/timeseries/TimeSeriesDispatchObserver.java
@@ -19,9 +19,14 @@
package org.apache.pinot.query.service.dispatch.timeseries;
import io.grpc.stub.StreamObserver;
-import java.util.function.Consumer;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
import org.apache.pinot.common.proto.Worker;
-import org.apache.pinot.query.routing.QueryServerInstance;
+import org.apache.pinot.query.runtime.timeseries.serde.TimeSeriesBlockSerde;
+import
org.apache.pinot.tsdb.planner.TimeSeriesPlanConstants.WorkerResponseMetadataKeys;
+import org.apache.pinot.tsdb.spi.series.TimeSeriesBlock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -30,37 +35,57 @@ import org.apache.pinot.query.routing.QueryServerInstance;
* engine integration.
*/
public class TimeSeriesDispatchObserver implements
StreamObserver<Worker.TimeSeriesResponse> {
- private final QueryServerInstance _serverInstance;
- private final Consumer<AsyncQueryTimeSeriesDispatchResponse> _callback;
+ /**
+ * Each server should send data for each leaf node once. This capacity
controls the size of the queue we use to
+ * buffer the data sent by the sender. This is set large enough that we
should never hit this for any practical
+ * use-case, while guarding us against bugs.
+ */
+ public static final int MAX_QUEUE_CAPACITY = 4096;
+ private static final Logger LOGGER =
LoggerFactory.getLogger(TimeSeriesDispatchObserver.class);
+ private final Map<String, BlockingQueue<Object>> _exchangeReceiversByPlanId;
- private Worker.TimeSeriesResponse _timeSeriesResponse;
-
- public TimeSeriesDispatchObserver(QueryServerInstance serverInstance,
- Consumer<AsyncQueryTimeSeriesDispatchResponse> callback) {
- _serverInstance = serverInstance;
- _callback = callback;
+ public TimeSeriesDispatchObserver(Map<String, BlockingQueue<Object>>
exchangeReceiversByPlanId) {
+ _exchangeReceiversByPlanId = exchangeReceiversByPlanId;
}
@Override
public void onNext(Worker.TimeSeriesResponse timeSeriesResponse) {
- _timeSeriesResponse = timeSeriesResponse;
+ if
(timeSeriesResponse.containsMetadata(WorkerResponseMetadataKeys.ERROR_TYPE)) {
+ String errorType =
timeSeriesResponse.getMetadataOrDefault(WorkerResponseMetadataKeys.ERROR_TYPE,
"");
+ String errorMessage =
timeSeriesResponse.getMetadataOrDefault(WorkerResponseMetadataKeys.ERROR_MESSAGE,
"");
+ onError(new Throwable(String.format("Error in server (type: %s): %s",
errorType, errorMessage)));
+ return;
+ }
+ String planId =
timeSeriesResponse.getMetadataMap().get(WorkerResponseMetadataKeys.PLAN_ID);
+ TimeSeriesBlock block = null;
+ Throwable error = null;
+ try {
+ block =
TimeSeriesBlockSerde.deserializeTimeSeriesBlock(timeSeriesResponse.getPayload().asReadOnlyByteBuffer());
+ } catch (Throwable t) {
+ error = t;
+ }
+ BlockingQueue<Object> receiverForPlanId =
_exchangeReceiversByPlanId.get(planId);
+ if (receiverForPlanId == null) {
+ String message = String.format("Receiver is not initialized for planId:
%s. Receivers exist only for planIds: %s",
+ planId, _exchangeReceiversByPlanId.keySet());
+ LOGGER.warn(message);
+ onError(new IllegalStateException(message));
+ } else {
+ if (!receiverForPlanId.offer(error != null ? error : block)) {
+ onError(new RuntimeException(String.format("Offer to receiver queue
(capacity=%s) for planId: %s failed",
+ receiverForPlanId.remainingCapacity(), planId)));
+ }
+ }
}
@Override
public void onError(Throwable throwable) {
- _callback.accept(
- new AsyncQueryTimeSeriesDispatchResponse(
- _serverInstance,
- Worker.TimeSeriesResponse.getDefaultInstance(),
- throwable));
+ for (BlockingQueue q : _exchangeReceiversByPlanId.values()) {
+ q.offer(throwable);
+ }
}
@Override
public void onCompleted() {
- _callback.accept(
- new AsyncQueryTimeSeriesDispatchResponse(
- _serverInstance,
- _timeSeriesResponse,
- null));
}
}
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/server/QueryServer.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/server/QueryServer.java
index 3d894baca9..e317add456 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/server/QueryServer.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/server/QueryServer.java
@@ -222,8 +222,7 @@ public class QueryServer extends
PinotQueryWorkerGrpc.PinotQueryWorkerImplBase {
@Override
public void submitTimeSeries(Worker.TimeSeriesQueryRequest request,
StreamObserver<Worker.TimeSeriesResponse> responseObserver) {
- String dispatchPlan = request.getDispatchPlan(0);
- _queryRunner.processTimeSeriesQuery(dispatchPlan,
request.getMetadataMap(), responseObserver);
+ _queryRunner.processTimeSeriesQuery(request.getDispatchPlanList(),
request.getMetadataMap(), responseObserver);
}
@Override
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesServerPlanVisitorTest.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesServerPlanVisitorTest.java
index 43b3496dfb..b30a82d165 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesServerPlanVisitorTest.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/timeseries/PhysicalTimeSeriesServerPlanVisitorTest.java
@@ -29,6 +29,9 @@ import
org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey;
import org.apache.pinot.tsdb.spi.AggInfo;
import org.apache.pinot.tsdb.spi.TimeBuckets;
import org.apache.pinot.tsdb.spi.plan.LeafTimeSeriesPlanNode;
+import org.apache.pinot.tsdb.spi.series.SimpleTimeSeriesBuilderFactory;
+import org.apache.pinot.tsdb.spi.series.TimeSeriesBuilderFactoryProvider;
+import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import static org.mockito.Mockito.mock;
@@ -38,8 +41,14 @@ import static org.testng.Assert.assertTrue;
public class PhysicalTimeSeriesServerPlanVisitorTest {
+ private static final String LANGUAGE = "m3ql";
private static final int DUMMY_DEADLINE_MS = 10_000;
+ @BeforeClass
+ public void setUp() {
+ TimeSeriesBuilderFactoryProvider.registerSeriesBuilderFactory(LANGUAGE,
new SimpleTimeSeriesBuilderFactory());
+ }
+
@Test
public void testCompileQueryContext() {
final String planId = "id";
@@ -52,14 +61,14 @@ public class PhysicalTimeSeriesServerPlanVisitorTest {
// Case-1: Without offset, simple column based group-by expression, simple
column based value, and non-empty filter.
{
TimeSeriesExecutionContext context =
- new TimeSeriesExecutionContext("m3ql", TimeBuckets.ofSeconds(1000L,
Duration.ofSeconds(10), 100),
- Collections.emptyMap(), DUMMY_DEADLINE_MS,
Collections.emptyMap());
+ new TimeSeriesExecutionContext(LANGUAGE,
TimeBuckets.ofSeconds(1000L, Duration.ofSeconds(10), 100),
+ DUMMY_DEADLINE_MS, Collections.emptyMap(),
Collections.emptyMap(), Collections.emptyMap());
LeafTimeSeriesPlanNode leafNode =
new LeafTimeSeriesPlanNode(planId, Collections.emptyList(),
tableName, timeColumn, TimeUnit.SECONDS, 0L,
filterExpr, "orderCount", aggInfo,
Collections.singletonList("cityName"));
QueryContext queryContext =
serverPlanVisitor.compileQueryContext(leafNode, context);
assertNotNull(queryContext.getTimeSeriesContext());
- assertEquals(queryContext.getTimeSeriesContext().getLanguage(), "m3ql");
+ assertEquals(queryContext.getTimeSeriesContext().getLanguage(),
LANGUAGE);
assertEquals(queryContext.getTimeSeriesContext().getOffsetSeconds(), 0L);
assertEquals(queryContext.getTimeSeriesContext().getTimeColumn(),
timeColumn);
assertEquals(queryContext.getTimeSeriesContext().getValueExpression().getIdentifier(),
"orderCount");
@@ -70,8 +79,8 @@ public class PhysicalTimeSeriesServerPlanVisitorTest {
// Case-2: With offset, complex group-by expression, complex value, and
non-empty filter
{
TimeSeriesExecutionContext context =
- new TimeSeriesExecutionContext("m3ql", TimeBuckets.ofSeconds(1000L,
Duration.ofSeconds(10), 100),
- Collections.emptyMap(), DUMMY_DEADLINE_MS,
Collections.emptyMap());
+ new TimeSeriesExecutionContext(LANGUAGE,
TimeBuckets.ofSeconds(1000L, Duration.ofSeconds(10), 100),
+ DUMMY_DEADLINE_MS, Collections.emptyMap(),
Collections.emptyMap(), Collections.emptyMap());
LeafTimeSeriesPlanNode leafNode =
new LeafTimeSeriesPlanNode(planId, Collections.emptyList(),
tableName, timeColumn, TimeUnit.SECONDS, 10L,
filterExpr, "orderCount*2", aggInfo,
Collections.singletonList("concat(cityName, stateName, '-')"));
@@ -80,7 +89,7 @@ public class PhysicalTimeSeriesServerPlanVisitorTest {
assertNotNull(queryContext.getGroupByExpressions());
assertEquals("concat(cityName,stateName,'-')",
queryContext.getGroupByExpressions().get(0).toString());
assertNotNull(queryContext.getTimeSeriesContext());
- assertEquals(queryContext.getTimeSeriesContext().getLanguage(), "m3ql");
+ assertEquals(queryContext.getTimeSeriesContext().getLanguage(),
LANGUAGE);
assertEquals(queryContext.getTimeSeriesContext().getOffsetSeconds(),
10L);
assertEquals(queryContext.getTimeSeriesContext().getTimeColumn(),
timeColumn);
assertEquals(queryContext.getTimeSeriesContext().getValueExpression().toString(),
"times(orderCount,'2')");
diff --git
a/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/TimeSeriesQueryEnvironment.java
b/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/TimeSeriesQueryEnvironment.java
index d061b21074..980c4f6bf3 100644
---
a/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/TimeSeriesQueryEnvironment.java
+++
b/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/TimeSeriesQueryEnvironment.java
@@ -19,20 +19,15 @@
package org.apache.pinot.tsdb.planner;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import java.lang.reflect.Constructor;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.function.Consumer;
import org.apache.pinot.common.config.provider.TableCache;
-import org.apache.pinot.common.request.BrokerRequest;
-import org.apache.pinot.common.request.DataSource;
-import org.apache.pinot.common.request.PinotQuery;
-import org.apache.pinot.common.request.QuerySource;
import org.apache.pinot.core.routing.RoutingManager;
-import org.apache.pinot.core.routing.RoutingTable;
-import org.apache.pinot.core.transport.ServerInstance;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.trace.RequestContext;
import org.apache.pinot.tsdb.planner.physical.TableScanVisitor;
@@ -43,8 +38,6 @@ import org.apache.pinot.tsdb.spi.RangeTimeSeriesRequest;
import org.apache.pinot.tsdb.spi.TimeSeriesLogicalPlanResult;
import org.apache.pinot.tsdb.spi.TimeSeriesLogicalPlanner;
import org.apache.pinot.tsdb.spi.plan.BaseTimeSeriesPlanNode;
-import org.apache.pinot.tsdb.spi.plan.LeafTimeSeriesPlanNode;
-import org.apache.pinot.tsdb.spi.plan.serde.TimeSeriesPlanSerde;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,55 +85,44 @@ public class TimeSeriesQueryEnvironment {
public TimeSeriesDispatchablePlan buildPhysicalPlan(RangeTimeSeriesRequest
timeSeriesRequest,
RequestContext requestContext, TimeSeriesLogicalPlanResult logicalPlan) {
- // Step-1: Find tables in the query.
- final Set<String> tableNames = new HashSet<>();
- findTableNames(logicalPlan.getPlanNode(), tableNames::add);
- Preconditions.checkState(tableNames.size() == 1,
- "Expected exactly one table name in the logical plan, got: %s",
- tableNames);
- String tableName = tableNames.iterator().next();
- // Step-2: Compute routing table assuming all segments are selected. This
is to perform the check to reject tables
- // that span across multiple servers.
- RoutingTable routingTable =
_routingManager.getRoutingTable(compileBrokerRequest(tableName),
- requestContext.getRequestId());
- Preconditions.checkState(routingTable != null,
- "Failed to get routing table for table: %s", tableName);
-
Preconditions.checkState(routingTable.getServerInstanceToSegmentsMap().size()
== 1,
- "Only support routing to a single server. Computed: %s",
- routingTable.getServerInstanceToSegmentsMap().size());
- var entry =
routingTable.getServerInstanceToSegmentsMap().entrySet().iterator().next();
- ServerInstance serverInstance = entry.getKey();
- // Step-3: Assign segments to the leaf plan nodes.
+ // Step-1: Assign segments to servers for each leaf node.
TableScanVisitor.Context scanVisitorContext =
TableScanVisitor.createContext(requestContext.getRequestId());
TableScanVisitor.INSTANCE.assignSegmentsToPlan(logicalPlan.getPlanNode(),
logicalPlan.getTimeBuckets(),
scanVisitorContext);
- return new TimeSeriesDispatchablePlan(timeSeriesRequest.getLanguage(),
- new TimeSeriesQueryServerInstance(serverInstance),
- TimeSeriesPlanSerde.serialize(logicalPlan.getPlanNode()),
logicalPlan.getTimeBuckets(),
- scanVisitorContext.getPlanIdToSegmentMap());
+ List<TimeSeriesQueryServerInstance> serverInstances =
scanVisitorContext.getQueryServers();
+ // Step-2: Create plan fragments.
+ List<BaseTimeSeriesPlanNode> fragments =
TimeSeriesPlanFragmenter.getFragments(
+ logicalPlan.getPlanNode(), serverInstances.size() == 1);
+ // Step-3: Compute number of servers each exchange node will receive data
from.
+ Map<String, Integer> numServersForExchangePlanNode =
computeNumServersForExchangePlanNode(serverInstances,
+ fragments, scanVisitorContext.getLeafIdToSegmentsByInstanceId());
+ return new TimeSeriesDispatchablePlan(timeSeriesRequest.getLanguage(),
serverInstances, fragments.get(0),
+ fragments.subList(1, fragments.size()), logicalPlan.getTimeBuckets(),
+ scanVisitorContext.getLeafIdToSegmentsByInstanceId(),
numServersForExchangePlanNode);
}
- public static void findTableNames(BaseTimeSeriesPlanNode planNode,
Consumer<String> tableNameConsumer) {
- if (planNode instanceof LeafTimeSeriesPlanNode) {
- LeafTimeSeriesPlanNode scanNode = (LeafTimeSeriesPlanNode) planNode;
- tableNameConsumer.accept(scanNode.getTableName());
- return;
+ private Map<String, Integer>
computeNumServersForExchangePlanNode(List<TimeSeriesQueryServerInstance>
serverInstances,
+ List<BaseTimeSeriesPlanNode> planNodes, Map<String, Map<String,
List<String>>> leafIdToSegmentsByInstanceId) {
+ // TODO(timeseries): Handle this gracefully and return an empty block.
+ Preconditions.checkState(!serverInstances.isEmpty(), "No servers selected
for the query");
+ if (serverInstances.size() == 1) {
+ // For single-server case, the broker fragment consists only of the
TimeSeriesExchangeNode.
+ return ImmutableMap.of(planNodes.get(0).getId(), 1);
}
- for (BaseTimeSeriesPlanNode childNode : planNode.getInputs()) {
- findTableNames(childNode, tableNameConsumer);
+ // For the multi-server case, the leafIdToSegmentsByInstanceId map already
has the information we need, but we
+ // just need to restructure it so that we can get number of servers by
planId.
+ Map<String, Set<String>> planIdToServers = new HashMap<>();
+ for (var entry : leafIdToSegmentsByInstanceId.entrySet()) {
+ String instanceId = entry.getKey();
+ for (var innerEntry : entry.getValue().entrySet()) {
+ String planId = innerEntry.getKey();
+ planIdToServers.computeIfAbsent(planId, (x) -> new
HashSet<>()).add(instanceId);
+ }
}
- }
-
- private BrokerRequest compileBrokerRequest(String tableName) {
- DataSource dataSource = new DataSource();
- dataSource.setTableName(tableName);
- PinotQuery pinotQuery = new PinotQuery();
- pinotQuery.setDataSource(dataSource);
- QuerySource querySource = new QuerySource();
- querySource.setTableName(tableName);
- BrokerRequest dummyRequest = new BrokerRequest();
- dummyRequest.setPinotQuery(pinotQuery);
- dummyRequest.setQuerySource(querySource);
- return dummyRequest;
+ Map<String, Integer> result = new HashMap<>();
+ for (var entry : planIdToServers.entrySet()) {
+ result.put(entry.getKey(), entry.getValue().size());
+ }
+ return result;
}
}
diff --git
a/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/physical/TableScanVisitor.java
b/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/physical/TableScanVisitor.java
index d9f80b54ac..3df75ce8ab 100644
---
a/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/physical/TableScanVisitor.java
+++
b/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/physical/TableScanVisitor.java
@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.pinot.common.request.BrokerRequest;
import org.apache.pinot.common.request.DataSource;
import org.apache.pinot.common.request.Expression;
@@ -29,6 +30,7 @@ import org.apache.pinot.common.request.PinotQuery;
import org.apache.pinot.common.request.QuerySource;
import org.apache.pinot.core.routing.RoutingManager;
import org.apache.pinot.core.routing.RoutingTable;
+import org.apache.pinot.core.transport.ServerInstance;
import org.apache.pinot.sql.parsers.CalciteSqlParser;
import org.apache.pinot.tsdb.spi.TimeBuckets;
import org.apache.pinot.tsdb.spi.plan.BaseTimeSeriesPlanNode;
@@ -54,12 +56,12 @@ public class TableScanVisitor {
compileBrokerRequest(sfpNode.getTableName(), filterExpression),
context._requestId);
Preconditions.checkNotNull(routingTable, "Failed to get routing table
for table: " + sfpNode.getTableName());
-
Preconditions.checkState(routingTable.getServerInstanceToSegmentsMap().size()
== 1,
- "Only support routing to a single server. Computed: %s",
- routingTable.getServerInstanceToSegmentsMap().size());
- var entry =
routingTable.getServerInstanceToSegmentsMap().entrySet().iterator().next();
- List<String> segments = entry.getValue().getLeft();
- context.getPlanIdToSegmentMap().put(sfpNode.getId(), segments);
+ for (var entry :
routingTable.getServerInstanceToSegmentsMap().entrySet()) {
+ ServerInstance serverInstance = entry.getKey();
+ List<String> segments = entry.getValue().getLeft();
+ context.getLeafIdToSegmentsByServer().computeIfAbsent(serverInstance,
(x) -> new HashMap<>())
+ .put(sfpNode.getId(), segments);
+ }
}
for (BaseTimeSeriesPlanNode childNode : planNode.getInputs()) {
assignSegmentsToPlan(childNode, timeBuckets, context);
@@ -71,15 +73,28 @@ public class TableScanVisitor {
}
public static class Context {
- private final Map<String, List<String>> _planIdToSegmentMap = new
HashMap<>();
+ private final Map<ServerInstance, Map<String, List<String>>>
_leafIdToSegmentsByServer = new HashMap<>();
private final Long _requestId;
public Context(Long requestId) {
_requestId = requestId;
}
- public Map<String, List<String>> getPlanIdToSegmentMap() {
- return _planIdToSegmentMap;
+ public List<TimeSeriesQueryServerInstance> getQueryServers() {
+ return
_leafIdToSegmentsByServer.keySet().stream().map(TimeSeriesQueryServerInstance::new).collect(
+ Collectors.toList());
+ }
+
+ public Map<String, Map<String, List<String>>>
getLeafIdToSegmentsByInstanceId() {
+ Map<String, Map<String, List<String>>> result = new HashMap<>();
+ for (var entry : _leafIdToSegmentsByServer.entrySet()) {
+ result.put(entry.getKey().getInstanceId(), entry.getValue());
+ }
+ return result;
+ }
+
+ Map<ServerInstance, Map<String, List<String>>>
getLeafIdToSegmentsByServer() {
+ return _leafIdToSegmentsByServer;
}
}
diff --git
a/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/physical/TimeSeriesDispatchablePlan.java
b/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/physical/TimeSeriesDispatchablePlan.java
index 6c64a396d8..8fa0152be7 100644
---
a/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/physical/TimeSeriesDispatchablePlan.java
+++
b/pinot-timeseries/pinot-timeseries-planner/src/main/java/org/apache/pinot/tsdb/planner/physical/TimeSeriesDispatchablePlan.java
@@ -20,42 +20,66 @@ package org.apache.pinot.tsdb.planner.physical;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.pinot.tsdb.spi.TimeBuckets;
+import org.apache.pinot.tsdb.spi.plan.BaseTimeSeriesPlanNode;
+import org.apache.pinot.tsdb.spi.plan.serde.TimeSeriesPlanSerde;
public class TimeSeriesDispatchablePlan {
- private final TimeSeriesQueryServerInstance _queryServerInstance;
+ private final List<TimeSeriesQueryServerInstance> _queryServerInstances;
private final String _language;
- private final String _serializedPlan;
+ private final BaseTimeSeriesPlanNode _brokerFragment;
+ private final List<BaseTimeSeriesPlanNode> _serverFragments;
private final TimeBuckets _timeBuckets;
- private final Map<String, List<String>> _planIdToSegments;
+ private final Map<String, Map<String, List<String>>>
_leafIdToSegmentsByInstanceId;
+ private final Map<String, Integer> _numInputServersForExchangePlanNode;
+ private final List<String> _serializedServerFragments;
- public TimeSeriesDispatchablePlan(String language,
TimeSeriesQueryServerInstance queryServerInstance,
- String serializedPlan, TimeBuckets timeBuckets, Map<String,
List<String>> planIdToSegments) {
+ public TimeSeriesDispatchablePlan(String language,
List<TimeSeriesQueryServerInstance> queryServerInstances,
+ BaseTimeSeriesPlanNode brokerFragment, List<BaseTimeSeriesPlanNode>
serverFragments,
+ TimeBuckets initialTimeBuckets, Map<String, Map<String, List<String>>>
leafIdToSegmentsByInstanceId,
+ Map<String, Integer> numInputServersForExchangePlanNode) {
_language = language;
- _queryServerInstance = queryServerInstance;
- _serializedPlan = serializedPlan;
- _timeBuckets = timeBuckets;
- _planIdToSegments = planIdToSegments;
+ _queryServerInstances = queryServerInstances;
+ _brokerFragment = brokerFragment;
+ _serverFragments = serverFragments;
+ _timeBuckets = initialTimeBuckets;
+ _leafIdToSegmentsByInstanceId = leafIdToSegmentsByInstanceId;
+ _numInputServersForExchangePlanNode = numInputServersForExchangePlanNode;
+ _serializedServerFragments =
serverFragments.stream().map(TimeSeriesPlanSerde::serialize).collect(
+ Collectors.toList());
}
public String getLanguage() {
return _language;
}
- public TimeSeriesQueryServerInstance getQueryServerInstance() {
- return _queryServerInstance;
+ public List<TimeSeriesQueryServerInstance> getQueryServerInstances() {
+ return _queryServerInstances;
}
- public String getSerializedPlan() {
- return _serializedPlan;
+ public BaseTimeSeriesPlanNode getBrokerFragment() {
+ return _brokerFragment;
+ }
+
+ public List<BaseTimeSeriesPlanNode> getServerFragments() {
+ return _serverFragments;
+ }
+
+ public List<String> getSerializedServerFragments() {
+ return _serializedServerFragments;
}
public TimeBuckets getTimeBuckets() {
return _timeBuckets;
}
- public Map<String, List<String>> getPlanIdToSegments() {
- return _planIdToSegments;
+ public Map<String, Map<String, List<String>>>
getLeafIdToSegmentsByInstanceId() {
+ return _leafIdToSegmentsByInstanceId;
+ }
+
+ public Map<String, Integer> getNumInputServersForExchangePlanNode() {
+ return _numInputServersForExchangePlanNode;
}
}
diff --git
a/pinot-timeseries/pinot-timeseries-spi/src/main/java/org/apache/pinot/tsdb/spi/series/TimeSeriesBuilderFactoryProvider.java
b/pinot-timeseries/pinot-timeseries-spi/src/main/java/org/apache/pinot/tsdb/spi/series/TimeSeriesBuilderFactoryProvider.java
index e82d3bdd44..b3189946ed 100644
---
a/pinot-timeseries/pinot-timeseries-spi/src/main/java/org/apache/pinot/tsdb/spi/series/TimeSeriesBuilderFactoryProvider.java
+++
b/pinot-timeseries/pinot-timeseries-spi/src/main/java/org/apache/pinot/tsdb/spi/series/TimeSeriesBuilderFactoryProvider.java
@@ -51,7 +51,7 @@ public class TimeSeriesBuilderFactoryProvider {
TimeSeriesBuilderFactory seriesBuilderFactory =
(TimeSeriesBuilderFactory) untypedSeriesBuilderFactory;
seriesBuilderFactory.init(pinotConfiguration.subset(
PinotTimeSeriesConfiguration.CONFIG_PREFIX + "." + language));
- FACTORY_MAP.put(language, seriesBuilderFactory);
+ FACTORY_MAP.putIfAbsent(language, seriesBuilderFactory);
} catch (Exception e) {
throw new RuntimeException(e);
}
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineQuickStart.java
index 0b00e2dad6..b64bec82f8 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineQuickStart.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineQuickStart.java
@@ -77,7 +77,7 @@ public class TimeSeriesEngineQuickStart extends Quickstart {
Preconditions.checkState(quickstartRunnerDir.mkdirs());
List<QuickstartTableRequest> quickstartTableRequests =
bootstrapStreamTableDirectories(quickstartTmpDir);
final QuickstartRunner runner =
- new QuickstartRunner(quickstartTableRequests, 1, 1, 1, 1,
quickstartRunnerDir, getConfigOverrides());
+ new QuickstartRunner(quickstartTableRequests, 1, 1, 2, 1,
quickstartRunnerDir, getConfigOverrides());
startKafka();
startAllDataStreams(_kafkaStarter, quickstartTmpDir);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]