This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/M4-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a52a15d960b37bd369a3616e679e9e9bf87ad62f Author: Lei Rui <[email protected]> AuthorDate: Wed Feb 1 16:20:00 2023 +0800 for pointsTraversed --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../apache/iotdb/db/engine/cache/ChunkCache.java | 2 +- .../dataset/groupby/LocalGroupByExecutor4CPV.java | 6 ++++++ .../org/apache/iotdb/db/service/TSServiceImpl.java | 2 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +- .../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 9 +++++++++ .../iotdb/tsfile/read/common}/IOMonitor.java | 23 +++++++++++++++++++++- .../iotdb/tsfile/read/reader/page/PageReader.java | 3 +++ 8 files changed, 44 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index f45461a069..cee0f8e24b 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -153,7 +153,7 @@ public class IoTDBConfig { private int flushWalThreshold = 10000; /** this variable set timestamp precision as millisecond, microsecond or nanosecond */ - private String timestampPrecision = "ms"; + private String timestampPrecision = "ns"; /** * The cycle when write ahead log is periodically forced to be written to disk(in milliseconds) If diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java index 02a1f5ab20..0e96e2a4cc 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.engine.cache; -import org.apache.iotdb.db.IOMonitor; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.query.control.FileReaderManager; @@ -27,6 +26,7 @@ import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; +import org.apache.iotdb.tsfile.read.common.IOMonitor; import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import com.github.benmanes.caffeine.cache.Caffeine; diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java index d961bbf02a..568d9a9cac 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java @@ -291,6 +291,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { .setDeleteIntervalList(chunkSuit4CPV.getChunkMetadata().getDeleteIntervalList()); } // TODO chunk data read operation (c): get all data points + // (c) get all data points chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV); // TODO check if empty if (chunkSuit4CPV.statistics.getCount() == 0) { @@ -376,6 +377,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { // WHICH WILL INTRODUCE BUGS! chunkSuit4CPV.setPageReader(pageReader); } + // (a) check existence of data point at a timestamp: isUpdate = chunkSuit4CPV.checkIfExist(candidateTimestamp); if (isUpdate) { // 提前结束对overlaps块的scan，因为已经找到一个update点证明candidate失效 break; @@ -492,6 +494,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { .setDeleteIntervalList(chunkSuit4CPV.getChunkMetadata().getDeleteIntervalList()); } // TODO chunk data read operation (c): get all data points + // (c) get all data points chunkSuit4CPV.getPageReader().updateBPTP(chunkSuit4CPV); // TODO check if empty if (chunkSuit4CPV.statistics.getCount() == 0) { @@ -582,6 +585,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { // WHICH WILL INTRODUCE BUGS! chunkSuit4CPV.setPageReader(pageReader); } + // (a) check existence of data point at a timestamp isUpdate = chunkSuit4CPV.checkIfExist(candidateTimestamp); if (isUpdate) { // 提前结束对overlaps块的scan，因为已经找到一个update点证明candidate失效 break; @@ -670,6 +674,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { susp_candidate.setPageReader(pageReader); } // TODO update FP equal to or after statistics.getEndTime + // (b) get the closest data point after or before a timestamp susp_candidate.updateFPwithTheClosetPointEqualOrAfter( susp_candidate.getStatistics().getStartTime()); // TODO DEBUG susp_candidate.setLazyLoad(false); // DO NOT FORGET THIS!!! @@ -778,6 +783,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { susp_candidate.setPageReader(pageReader); } // TODO update FP equal to or after statistics.getEndTime + // (b) get the closest data point after or before a timestamp susp_candidate.updateLPwithTheClosetPointEqualOrBefore( susp_candidate.getStatistics().getEndTime()); // TODO DEBUG susp_candidate.setLazyLoad(false); // TODO DO NOT FORGET THIS!!! diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java index 4b6db1bbaf..00c01bb37d 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.service; -import org.apache.iotdb.db.IOMonitor; import org.apache.iotdb.db.auth.AuthException; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.auth.authorizer.BasicAuthorizer; @@ -128,6 +127,7 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.common.IOMonitor; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index 7d2b9dc5cd..fd7b6f4cb1 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.utils; -import org.apache.iotdb.db.IOMonitor; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -34,6 +33,7 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; +import org.apache.iotdb.tsfile.read.common.IOMonitor; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.filter.basic.Filter; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java index 55116a7619..d61d14e193 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java @@ -254,13 +254,16 @@ public class ChunkSuit4CPV { if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { estimatedPos++; + IOMonitor.incPointsTravered(); } } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { estimatedPos--; + IOMonitor.incPointsTravered(); } if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { estimatedPos++; + IOMonitor.incPointsTravered(); } // else equal } // else equal this.startPos = estimatedPos; // note this @@ -315,13 +318,16 @@ public class ChunkSuit4CPV { if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { estimatedPos--; + IOMonitor.incPointsTravered(); } } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { estimatedPos++; + IOMonitor.incPointsTravered(); } if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { estimatedPos--; + IOMonitor.incPointsTravered(); } // else equal } // else equal this.endPos = estimatedPos; // note this @@ -375,13 +381,16 @@ public class ChunkSuit4CPV { if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { estimatedPos--; + IOMonitor.incPointsTravered(); } } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { estimatedPos++; + IOMonitor.incPointsTravered(); } if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { estimatedPos--; + IOMonitor.incPointsTravered(); } // else equal } // else equal diff --git a/server/src/main/java/org/apache/iotdb/db/IOMonitor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor.java similarity index 89% rename from server/src/main/java/org/apache/iotdb/db/IOMonitor.java rename to tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor.java index 698aa10cb3..3bfed74089 100644 --- a/server/src/main/java/org/apache/iotdb/db/IOMonitor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db; +package org.apache.iotdb.tsfile.read.common; import java.util.ArrayList; import java.util.List; @@ -34,6 +34,8 @@ public class IOMonitor { public static long totalTime; public static String sql; + public static long pointsTraversed; + public static List<Long> metaIOTimes = new ArrayList<>(); public static List<Long> dataIOTimes = new ArrayList<>(); public static List<Long> readMemChunkTimes = new ArrayList<>(); @@ -44,6 +46,8 @@ public class IOMonitor { public static List<String> sqls = new ArrayList<>(); public static List<Long> totalTimes = new ArrayList<>(); + public static List<Long> pointsTraversedList = new ArrayList<>(); + public static boolean isSet = false; public static void incReadMemChunkTime(long v) { @@ -58,11 +62,20 @@ public class IOMonitor { readMemChunkNum = 0; } + public static void resetPointsTraversed() { + pointsTraversedList.add(pointsTraversed); + pointsTraversed = 0; + } + public static void incMeta(long v) { metaIOTime += v; metaIONum++; } + public static void incPointsTravered() { + pointsTraversed++; + } + private static void resetMeta() { metaIOTimes.add(metaIOTime); metaIONums.add(metaIONum); @@ -109,6 +122,7 @@ public class IOMonitor { resetReadMemChunkTime(); sqls.add(sql); sql = null; + resetPointsTraversed(); } // private static double getAvg(List<Long> vals) { @@ -148,6 +162,9 @@ public class IOMonitor { dataIOTimes.clear(); sqls.clear(); totalTimes.clear(); + + pointsTraversedList.clear(); + pointsTraversed = 0; } public static void finish() { @@ -177,6 +194,8 @@ public class IOMonitor { + readMemChunkNums.get(i) + "\t total: \t" + totalTimes.get(i) + + "\t pointsTraversed: \t" + + pointsTraversedList.get(i) + "\n"; } ret += @@ -194,6 +213,8 @@ public class IOMonitor { + getSumInteger(readMemChunkNums) + "\t avg total time: \t" + getSumLong(totalTimes) + + "\t points traversed: \t" + + getSumLong(pointsTraversedList) + "\t isSet: \t" + isSet; return ret; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java index cd67765f45..f3e300a7ea 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java @@ -30,6 +30,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.common.BatchDataFactory; import org.apache.iotdb.tsfile.read.common.ChunkSuit4CPV; +import org.apache.iotdb.tsfile.read.common.IOMonitor; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.operator.AndFilter; @@ -136,6 +137,7 @@ public class PageReader implements IPageReader { // TODO update FP,LP with the help of stepRegress index. BP/TP not update here. int FP_pos = -1; int LP_pos = -1; + // (b) get the closest data point after or before a timestamp if (leftEndIncluded > chunkSuit4CPV.statistics.getStartTime()) { FP_pos = chunkSuit4CPV.updateFPwithTheClosetPointEqualOrAfter(leftEndIncluded); } @@ -182,6 +184,7 @@ public class PageReader implements IPageReader { // [startPos,endPos] definitely for curStartTime interval, thanks to split4CPV int count = 0; // update here, not in statistics for (int pos = chunkSuit4CPV.startPos; pos <= chunkSuit4CPV.endPos; pos++) { + IOMonitor.incPointsTravered(); long timestamp = timeBuffer.getLong(pos * 8); switch (dataType) { case INT64:
