This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch MasterQueryDebug in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bb0c0b90c47e970e08d42336be6e50bf8857e77e Author: JackieTien97 <[email protected]> AuthorDate: Mon Mar 15 17:46:53 2021 +0800 Add Explain Query Support --- .../apache/iotdb/db/engine/cache/ChunkCache.java | 6 ++++- .../db/engine/cache/TimeSeriesMetadataCache.java | 15 ++++++++---- .../iotdb/db/engine/modification/Deletion.java | 16 +++++++++++++ .../engine/storagegroup/StorageGroupProcessor.java | 8 ++++--- .../db/engine/storagegroup/TsFileResource.java | 9 ++++---- .../apache/iotdb/db/qp/physical/PhysicalPlan.java | 10 ++++++++ .../iotdb/db/query/context/QueryContext.java | 11 +++++++++ .../db/query/reader/chunk/DiskChunkLoader.java | 9 +++++--- .../chunk/metadata/DiskChunkMetadataLoader.java | 27 +++++++++++++++++++++- .../org/apache/iotdb/db/service/TSServiceImpl.java | 8 +++---- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 3 ++- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 4 ++-- .../tsfile/file/metadata/TimeseriesMetadata.java | 25 ++++++++++++++++++++ 13 files changed, 127 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java index f3462c6..556075f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java @@ -89,6 +89,10 @@ public class ChunkCache { } public Chunk get(ChunkMetadata chunkMetaData) throws IOException { + return get(chunkMetaData, false); + } + + public Chunk get(ChunkMetadata chunkMetaData, boolean debug) throws IOException { if (!CACHE_ENABLE) { TsFileSequenceReader reader = FileReaderManager.getInstance() @@ -134,7 +138,7 @@ public class ChunkCache { } } - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info("get chunk from cache whose meta data is: " + chunkMetaData); } return new Chunk( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java index d6ad1f4..0fe95da 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java @@ -122,9 +122,14 @@ public class TimeSeriesMetadataCache { return TimeSeriesMetadataCache.TimeSeriesMetadataCacheHolder.INSTANCE; } - @SuppressWarnings("squid:S1860") // Suppress synchronize warning public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set<String> allSensors) throws IOException { + return get(key, allSensors, false); + } + + @SuppressWarnings("squid:S1860") // Suppress synchronize warning + public TimeseriesMetadata get( + TimeSeriesMetadataCacheKey key, Set<String> allSensors, boolean debug) throws IOException { if (!CACHE_ENABLE) { // bloom filter part TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true); @@ -150,7 +155,7 @@ public class TimeSeriesMetadataCache { cacheHitNum.incrementAndGet(); printCacheLog(true); } else { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info( "Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath); DEBUG_LOGGER.info("Device: {}, all sensors: {}", key.device, allSensors); @@ -174,7 +179,7 @@ public class TimeSeriesMetadataCache { TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true); BloomFilter bloomFilter = reader.readBloomFilter(); if (bloomFilter != null && !bloomFilter.contains(path.getFullPath())) { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key); } return null; @@ -202,12 +207,12 @@ public class TimeSeriesMetadataCache { } } if (timeseriesMetadata == null) { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info("The file doesn't have this time series {}.", key); } return null; } else { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info( "Get timeseries: {}.{} metadata in file: {} from cache: {}.", key.device, diff --git a/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java b/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java index 5ed33b3..5d41dbe 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java @@ -88,4 +88,20 @@ public class Deletion extends Modification { public int hashCode() { return Objects.hash(super.hashCode(), startTime, endTime); } + + @Override + public String toString() { + return "Deletion{" + + "startTime=" + + startTime + + ", endTime=" + + endTime + + ", type=" + + type + + ", path=" + + path + + ", fileOffset=" + + fileOffset + + '}'; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java index b5b76df..d2720f4 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java @@ -1599,7 +1599,7 @@ public class StorageGroupProcessor { boolean isSeq) throws MetadataException { - if (config.isDebugOn()) { + if (context.isDebug()) { DEBUG_LOGGER.info( "Path: {}.{}, get tsfile list: {} isSeq: {} timefilter: {}", deviceId.getFullPath(), @@ -1618,7 +1618,8 @@ public class StorageGroupProcessor { // for upgrade files and old files must be closed for (TsFileResource tsFileResource : upgradeTsFileResources) { - if (!tsFileResource.isSatisfied(deviceId.getFullPath(), timeFilter, isSeq, dataTTL)) { + if (!tsFileResource.isSatisfied( + deviceId.getFullPath(), timeFilter, isSeq, dataTTL, context.isDebug())) { continue; } closeQueryLock.readLock().lock(); @@ -1630,7 +1631,8 @@ public class StorageGroupProcessor { } for (TsFileResource tsFileResource : tsFileResources) { - if (!tsFileResource.isSatisfied(deviceId.getFullPath(), timeFilter, isSeq, dataTTL)) { + if (!tsFileResource.isSatisfied( + deviceId.getFullPath(), timeFilter, isSeq, dataTTL, context.isDebug())) { continue; } closeQueryLock.readLock().lock(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java index 8a9cd3a..86ea847 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java @@ -541,9 +541,10 @@ public class TsFileResource { } /** @return true if the device is contained in the TsFile and it lives beyond TTL */ - public boolean isSatisfied(String deviceId, Filter timeFilter, boolean isSeq, long ttl) { + public boolean isSatisfied( + String deviceId, Filter timeFilter, boolean isSeq, long ttl, boolean debug) { if (!getDevices().contains(deviceId)) { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info( "Path: {} file {} is not satisfied because of no device!", deviceId, file); } @@ -554,7 +555,7 @@ public class TsFileResource { long endTime = closed || !isSeq ? getEndTime(deviceId) : Long.MAX_VALUE; if (!isAlive(endTime, ttl)) { - if (config.isDebugOn()) { + if (debug) { DEBUG_LOGGER.info("Path: {} file {} is not satisfied because of ttl!", deviceId, file); } return false; @@ -562,7 +563,7 @@ public class TsFileResource { if (timeFilter != null) { boolean res = timeFilter.satisfyStartEndTime(startTime, endTime); - if (config.isDebugOn() && !res) { + if (debug && !res) { DEBUG_LOGGER.info( "Path: {} file {} is not satisfied because of time filter!", deviceId, fsFactory); } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java index 5a5ad27..16719c4 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java @@ -75,6 +75,8 @@ public abstract class PhysicalPlan { // a bridge from a cluster raft log to a physical plan protected long index; + private boolean debug; + /** whether the plan can be split into more than one Plans. Only used in the cluster mode. */ public boolean canBeSplit() { return canBeSplit; @@ -117,6 +119,14 @@ public abstract class PhysicalPlan { isQuery = query; } + public boolean isDebug() { + return debug; + } + + public void setDebug(boolean debug) { + this.debug = debug; + } + /** * Serialize the plan into the given buffer. All necessary fields will be serialized. * diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java index 3febfa9..326e71f 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java +++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java @@ -49,12 +49,19 @@ public class QueryContext { private long queryTimeLowerBound = Long.MIN_VALUE; + private boolean debug; + public QueryContext() {} public QueryContext(long queryId) { this.queryId = queryId; } + public QueryContext(long queryId, boolean debug) { + this.queryId = queryId; + this.debug = debug; + } + /** * Find the modifications of timeseries 'path' in 'modFile'. If they are not in the cache, read * them from 'modFile' and put then into the cache. @@ -87,6 +94,10 @@ public class QueryContext { return queryId; } + public boolean isDebug() { + return debug; + } + public long getQueryTimeLowerBound() { return queryTimeLowerBound; } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java index 9c02149..36619e2 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.query.reader.chunk; import org.apache.iotdb.db.engine.cache.ChunkCache; +import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.controller.IChunkLoader; @@ -29,13 +30,15 @@ import java.io.IOException; /** To read one chunk from disk, and only used in iotdb server module */ public class DiskChunkLoader implements IChunkLoader { - public DiskChunkLoader() { - // do nothing + private final QueryContext context; + + public DiskChunkLoader(QueryContext context) { + this.context = context; } @Override public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException { - return ChunkCache.getInstance().get(chunkMetaData); + return ChunkCache.getInstance().get(chunkMetaData, context.isDebug()); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java index ebb9521..cdaf128 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java @@ -29,6 +29,9 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader; import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.List; public class DiskChunkMetadataLoader implements IChunkMetadataLoader { @@ -39,6 +42,8 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { // time filter or value filter, only used to check time range private final Filter filter; + private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG"); + public DiskChunkMetadataLoader( TsFileResource resource, PartialPath seriesPath, QueryContext context, Filter filter) { this.resource = resource; @@ -69,6 +74,12 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { metadata.setVersion(resource.getVersion()); } } + + if (context.isDebug()) { + DEBUG_LOGGER.info("After removed by filter Chunk meta data list is: "); + chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString())); + } + return chunkMetadataList; } @@ -80,9 +91,23 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { List<Modification> pathModifications = context.getPathModifications(resource.getModFile(), seriesPath); + if (context.isDebug()) { + DEBUG_LOGGER.info( + "Modifications size is {} for file Path: {} ", + pathModifications.size(), + resource.getTsFilePath()); + pathModifications.forEach(c -> DEBUG_LOGGER.info(c.toString())); + } + if (!pathModifications.isEmpty()) { QueryUtils.modifyChunkMetaData(chunkMetadataList, pathModifications); } + + if (context.isDebug()) { + DEBUG_LOGGER.info("After modification Chunk meta data list is: "); + chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString())); + } + // it is ok, even if it is not thread safe, because the cost of creating a DiskChunkLoader is // very cheap. chunkMetadataList.forEach( @@ -90,7 +115,7 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { if (chunkMetadata.getChunkLoader() == null) { chunkMetadata.setFilePath(resource.getTsFilePath()); chunkMetadata.setClosed(resource.isClosed()); - chunkMetadata.setChunkLoader(new DiskChunkLoader()); + chunkMetadata.setChunkLoader(new DiskChunkLoader(context)); } }); } diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java index edc0725..90987c5 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java @@ -728,7 +728,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { if (costTime >= config.getSlowQueryThreshold()) { SLOW_SQL_LOGGER.info("Cost: {} ms, sql is {}", costTime, statement); } - if (config.isDebugOn()) { + if (plan.isDebug()) { SLOW_SQL_LOGGER.info( "ChunkCache used memory proportion: {}\n" + "TimeSeriesMetadataCache used memory proportion: {}", @@ -1029,14 +1029,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { throws QueryProcessException, QueryFilterOptimizationException, StorageEngineException, IOException, MetadataException, SQLException, TException, InterruptedException { - QueryContext context = genQueryContext(queryId); + QueryContext context = genQueryContext(queryId, physicalPlan.isDebug()); QueryDataSet queryDataSet = executor.processQuery(physicalPlan, context); queryId2DataSet.put(queryId, queryDataSet); return queryDataSet; } - protected QueryContext genQueryContext(long queryId) { - return new QueryContext(queryId); + protected QueryContext genQueryContext(long queryId, boolean debug) { + return new QueryContext(queryId, debug); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index a829b69..0a55b54 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@ -104,7 +104,8 @@ public class FileLoaderUtils { resource.getTsFilePath(), seriesPath.getDevice(), seriesPath.getMeasurement()), - allSensors); + allSensors, + context.isDebug()); if (timeSeriesMetadata != null) { timeSeriesMetadata.setChunkMetadataLoader( new DiskChunkMetadataLoader(resource, seriesPath, context, filter)); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java index de6681d..07260bc 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java @@ -93,8 +93,8 @@ public class ChunkMetadata implements Accountable { @Override public String toString() { return String.format( - "measurementId: %s, datatype: %s, version: %d, " + "Statistics: %s, deleteIntervalList: %s", - measurementUid, tsDataType, version, statistics, deleteIntervalList); + "measurementId: %s, datatype: %s, version: %d, Statistics: %s, deleteIntervalList: %s, filePath: %s", + measurementUid, tsDataType, version, statistics, deleteIntervalList, filePath); } public long getNumOfPoints() { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java index 20dbf05..7344c2e 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java @@ -231,4 +231,29 @@ public class TimeseriesMetadata implements Accountable { public void setChunkMetadataList(ArrayList<ChunkMetadata> chunkMetadataList) { this.chunkMetadataList = chunkMetadataList; } + + @Override + public String toString() { + return "TimeseriesMetadata{" + + "startOffsetOfChunkMetaDataList=" + + startOffsetOfChunkMetaDataList + + ", timeSeriesMetadataType=" + + timeSeriesMetadataType + + ", chunkMetaDataListDataSize=" + + chunkMetaDataListDataSize + + ", measurementId='" + + measurementId + + '\'' + + ", dataType=" + + dataType + + ", statistics=" + + statistics + + ", modified=" + + modified + + ", isSeq=" + + isSeq + + ", chunkMetadataList=" + + chunkMetadataList + + '}'; + } }
