This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new c09bb166883 Optimize query execution in easy count(*) cases
c09bb166883 is described below
commit c09bb166883e1b37c172c1e6e60d3fc71ef9de9d
Author: Jackie Tien <[email protected]>
AuthorDate: Tue Mar 18 15:22:18 2025 +0800
Optimize query execution in easy count(*) cases
---
.../fragment/FragmentInstanceContext.java | 38 +++++++++++-----
.../buffer/TimeSeriesMetadataCache.java | 3 +-
.../dataregion/VirtualDataRegion.java | 2 +-
.../dataregion/tsfile/TsFileResource.java | 50 ++++++++++++----------
.../tsfile/timeindex/ArrayDeviceTimeIndex.java | 12 ++++--
5 files changed, 66 insertions(+), 39 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index 561eb432c6a..13fd46ba540 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.execution.fragment;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
+import org.apache.iotdb.commons.path.AlignedFullPath;
import org.apache.iotdb.commons.path.IFullPath;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.exception.query.QueryProcessException;
@@ -67,6 +68,7 @@ import java.util.stream.Collectors;
import static
org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet.BLOCK_QUEUED_TIME;
import static
org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet.READY_QUEUED_TIME;
+import static
org.apache.iotdb.db.storageengine.dataregion.VirtualDataRegion.EMPTY_QUERY_DATA_SOURCE;
import static org.apache.iotdb.db.utils.ErrorHandlingUtils.getRootCause;
import static org.apache.iotdb.rpc.TSStatusCode.DATE_OUT_OF_RANGE;
@@ -450,24 +452,40 @@ public class FragmentInstanceContext extends QueryContext
{
public void initQueryDataSource(List<IFullPath> sourcePaths) throws
QueryProcessException {
long startTime = System.nanoTime();
- if (sourcePaths == null) {
+ if (sourcePaths == null || sourcePaths.isEmpty()) {
+ this.sharedQueryDataSource = EMPTY_QUERY_DATA_SOURCE;
return;
}
- dataRegion.readLock();
- try {
- List<IFullPath> pathList = new ArrayList<>();
+
+ IDeviceID singleDeviceId = null;
+ if (sourcePaths.size() == 1) {
+ singleDeviceId = sourcePaths.get(0).getDeviceId();
+ } else {
Set<IDeviceID> selectedDeviceIdSet = new HashSet<>();
- for (IFullPath path : sourcePaths) {
- pathList.add(path);
- selectedDeviceIdSet.add(path.getDeviceId());
+ for (IFullPath sourcePath : sourcePaths) {
+ if (sourcePath instanceof AlignedFullPath) {
+ singleDeviceId = null;
+ break;
+ } else {
+ singleDeviceId = sourcePath.getDeviceId();
+ selectedDeviceIdSet.add(singleDeviceId);
+ if (selectedDeviceIdSet.size() > 1) {
+ singleDeviceId = null;
+ break;
+ }
+ }
}
+ }
+
+ dataRegion.readLock();
+ try {
this.sharedQueryDataSource =
dataRegion.query(
- pathList,
+ sourcePaths,
// when all the selected series are under the same device, the
QueryDataSource will be
// filtered according to timeIndex
- selectedDeviceIdSet.size() == 1 ?
selectedDeviceIdSet.iterator().next() : null,
+ singleDeviceId,
this,
// time filter may be stateful, so we need to copy it
globalTimeFilter != null ? globalTimeFilter.copy() : null,
@@ -479,7 +497,7 @@ public class FragmentInstanceContext extends QueryContext {
closedFilePaths = new HashSet<>();
unClosedFilePaths = new HashSet<>();
addUsedFilesForQuery((QueryDataSource) sharedQueryDataSource);
- ((QueryDataSource)
sharedQueryDataSource).setSingleDevice(selectedDeviceIdSet.size() == 1);
+ ((QueryDataSource)
sharedQueryDataSource).setSingleDevice(singleDeviceId != null);
}
} finally {
setInitQueryDataSourceCost(System.nanoTime() - startTime);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
index 3fb377fabba..f2cd55c5e7d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
@@ -126,8 +126,8 @@ public class TimeSeriesMetadataCache {
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
boolean cacheHit = true;
try {
- String deviceStringFormat = key.device.toString();
if (!CACHE_ENABLE) {
+ String deviceStringFormat = key.device.toString();
cacheHit = false;
// bloom filter part
@@ -158,6 +158,7 @@ public class TimeSeriesMetadataCache {
DEBUG_LOGGER.info("Cache miss: {}.{} in file: {}", key.device,
key.measurement, filePath);
DEBUG_LOGGER.info("Device: {}, all sensors: {}", key.device,
allSensors);
}
+ String deviceStringFormat = key.device.toString();
// allow for the parallelism of different devices
synchronized (
devices.computeIfAbsent(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java
index 8131453937a..a104ee32e68 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java
@@ -41,7 +41,7 @@ public class VirtualDataRegion implements IDataRegionForQuery
{
private static final String VIRTUAL_DB_NAME = "root.__virtual";
- private static final QueryDataSource EMPTY_QUERY_DATA_SOURCE =
+ public static final QueryDataSource EMPTY_QUERY_DATA_SOURCE =
new QueryDataSource(Collections.emptyList(), Collections.emptyList());
private static final QueryDataSourceForRegionScan
EMPTY_REGION_QUERY_DATA_SOURCE =
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index c1a6e1826f4..a90aaf72474 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -634,10 +634,10 @@ public class TsFileResource implements PersistentResource
{
// cannot use FileTimeIndex
public long getOrderTimeForSeq(IDeviceID deviceId, boolean ascending) {
- if (timeIndex instanceof ArrayDeviceTimeIndex &&
!definitelyNotContains(deviceId)) {
- // checked above
- //noinspection OptionalGetWithoutIsPresent
- return ascending ? getStartTime(deviceId).get() :
getEndTime(deviceId).get();
+ if (timeIndex instanceof ArrayDeviceTimeIndex) {
+ return ascending
+ ? timeIndex.getStartTime(deviceId).orElse(Long.MIN_VALUE)
+ : timeIndex.getEndTime(deviceId).orElse(Long.MAX_VALUE);
} else {
return ascending ? Long.MIN_VALUE : Long.MAX_VALUE;
}
@@ -645,12 +645,15 @@ public class TsFileResource implements PersistentResource
{
// can use FileTimeIndex
public long getOrderTimeForUnseq(IDeviceID deviceId, boolean ascending) {
- if (!definitelyNotContains(deviceId)) {
- // checked above
- //noinspection OptionalGetWithoutIsPresent
- return ascending ? getStartTime(deviceId).get() :
getEndTime(deviceId).get();
+ if (timeIndex instanceof ArrayDeviceTimeIndex) {
+ if (ascending) {
+ return timeIndex.getStartTime(deviceId).orElse(Long.MIN_VALUE);
+ } else {
+ return timeIndex.getEndTime(deviceId).orElse(Long.MAX_VALUE);
+ }
} else {
- return ascending ? Long.MIN_VALUE : Long.MAX_VALUE;
+ // FileTimeIndex
+ return ascending ? getFileStartTime() : getFileEndTime();
}
}
@@ -1000,21 +1003,22 @@ public class TsFileResource implements
PersistentResource {
return false;
}
- // check above
- long startTime = getStartTime(deviceId).get();
- long endTime = isClosed() || !isSeq ? getEndTime(deviceId).get() :
Long.MAX_VALUE;
- if (startTime > endTime) {
- // startTime > endTime indicates that there is something wrong with this
TsFile. Return false
- // directly, or it may lead to infinite loop in
GroupByMonthFilter#getTimePointPosition.
- LOGGER.warn(
- "startTime[{}] of TsFileResource[{}] is greater than its
endTime[{}]",
- startTime,
- this,
- endTime);
- return false;
- }
-
if (timeFilter != null) {
+ // check above
+ long startTime = getStartTime(deviceId).get();
+ long endTime = isClosed() || !isSeq ? getEndTime(deviceId).get() :
Long.MAX_VALUE;
+ if (startTime > endTime) {
+ // startTime > endTime indicates that there is something wrong with
this TsFile. Return
+ // false
+ // directly, or it may lead to infinite loop in
GroupByMonthFilter#getTimePointPosition.
+ LOGGER.warn(
+ "startTime[{}] of TsFileResource[{}] is greater than its
endTime[{}]",
+ startTime,
+ this,
+ endTime);
+ return false;
+ }
+
boolean res = timeFilter.satisfyStartEndTime(startTime, endTime);
if (debug && !res) {
DEBUG_LOGGER.info(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java
index 7247eb93f61..f8503c79af5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java
@@ -375,18 +375,22 @@ public class ArrayDeviceTimeIndex implements ITimeIndex {
@Override
public Optional<Long> getStartTime(IDeviceID deviceId) {
- if (!deviceToIndex.containsKey(deviceId)) {
+ Integer index = deviceToIndex.get(deviceId);
+ if (index == null) {
return Optional.empty();
+ } else {
+ return Optional.of(startTimes[index]);
}
- return Optional.of(startTimes[deviceToIndex.get(deviceId)]);
}
@Override
public Optional<Long> getEndTime(IDeviceID deviceId) {
- if (!deviceToIndex.containsKey(deviceId)) {
+ Integer index = deviceToIndex.get(deviceId);
+ if (index == null) {
return Optional.empty();
+ } else {
+ return Optional.of(endTimes[index]);
}
- return Optional.of(endTimes[deviceToIndex.get(deviceId)]);
}
@Override
