This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/M4-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b24dbd33b034bf71f505476f9b9f97e481ef318f Author: Lei Rui <[email protected]> AuthorDate: Tue Jun 13 22:08:35 2023 +0800 [important] four-layer metrics design done, from client, server execute&fetch, dataset&executor, basic ops ABCD, improved D search array 4 functions --- .../dataset/groupby/LocalGroupByExecutor4CPV.java | 18 ++ .../iotdb/tsfile/read/common/ChunkSuit4CPV.java | 40 ++- .../iotdb/tsfile/read/common/IOMonitor2.java | 282 +++++++++++++++++---- .../iotdb/tsfile/read/reader/page/PageReader.java | 3 + 4 files changed, 289 insertions(+), 54 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java index b4c59643688..92026420e17 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor4CPV.java @@ -38,6 +38,8 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.IntegerStatistics; import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.ChunkSuit4CPV; +import org.apache.iotdb.tsfile.read.common.IOMonitor2; +import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.GroupByFilter; import org.apache.iotdb.tsfile.read.filter.basic.Filter; @@ -93,6 +95,7 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { TsFileFilter fileFilter, boolean ascending) throws StorageEngineException, QueryProcessException { + long start = System.nanoTime(); this.tsDataType = dataType; // this.mergeReader = new PriorityMergeReader(); @@ -225,6 +228,8 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { } catch (IOException e) { throw new QueryProcessException(e.getMessage()); } + + IOMonitor2.addMeasure(Operation.M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS, System.nanoTime() - start); } @Override @@ -309,16 +314,29 @@ public class LocalGroupByExecutor4CPV implements GroupByExecutor { result.reset(); } + long start = System.nanoTime(); getCurrentChunkListFromFutureChunkList(curStartTime, curEndTime, startTime, endTime, interval); + IOMonitor2.addMeasure(Operation.M4_LSM_MERGE_M4_TIME_SPAN, System.nanoTime() - start); if (currentChunkList.size() == 0) { return results; } + start = System.nanoTime(); calculateFirstPoint(currentChunkList, startTime, endTime, interval, curStartTime); + IOMonitor2.addMeasure(Operation.M4_LSM_FP, System.nanoTime() - start); + + start = System.nanoTime(); calculateLastPoint(currentChunkList, startTime, endTime, interval, curStartTime); + IOMonitor2.addMeasure(Operation.M4_LSM_LP, System.nanoTime() - start); + + start = System.nanoTime(); calculateBottomPoint(currentChunkList, startTime, endTime, interval, curStartTime); + IOMonitor2.addMeasure(Operation.M4_LSM_BP, System.nanoTime() - start); + + start = System.nanoTime(); calculateTopPoint(currentChunkList, startTime, endTime, interval, curStartTime); + IOMonitor2.addMeasure(Operation.M4_LSM_TP, System.nanoTime() - start); return results; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java index 0834348a268..67520439d22 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/ChunkSuit4CPV.java @@ -29,6 +29,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics; import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.file.metadata.statistics.StepRegress; +import org.apache.iotdb.tsfile.read.common.IOMonitor2.Operation; import org.apache.iotdb.tsfile.read.reader.page.PageReader; import java.io.IOException; @@ -245,10 +246,12 @@ public class ChunkSuit4CPV { * @return the position of the point, starting from 0 */ public int updateFPwithTheClosetPointEqualOrAfter(long targetTimestamp) throws IOException { + long start = System.nanoTime(); + int estimatedPos; if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) { StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress(); // infer position starts from 1, so minus 1 here - int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1; + estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1; // search from estimatePos in the timeBuffer to find the closet timestamp equal to or larger // than the given timestamp @@ -256,15 +259,18 @@ public class ChunkSuit4CPV { while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { estimatedPos++; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; } } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { estimatedPos--; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; } if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { estimatedPos++; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; } // else equal } // else equal this.startPos = estimatedPos; // note this @@ -298,16 +304,16 @@ public class ChunkSuit4CPV { default: throw new IOException("Unsupported data type!"); } - return estimatedPos; } else { // search from estimatePos in the timeBuffer to find the closet timestamp equal to or larger // than the given timestamp - int estimatedPos = -1; + estimatedPos = -1; pageReader.timeBuffer.position(0); pageReader.valueBuffer.position(pageReader.timeBufferLength); while (pageReader.timeBuffer.remaining() > 0) { estimatedPos++; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; long t = pageReader.timeBuffer.getLong(); if (t >= targetTimestamp) { break; @@ -346,6 +352,8 @@ public class ChunkSuit4CPV { } return estimatedPos; } + IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_b_genFP, System.nanoTime() - start); + return estimatedPos; } /** @@ -356,10 +364,12 @@ public class ChunkSuit4CPV { * @return the position of the point, starting from 0 */ public int updateLPwithTheClosetPointEqualOrBefore(long targetTimestamp) throws IOException { + long start = System.nanoTime(); + int estimatedPos; if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) { StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress(); // infer position starts from 1, so minus 1 here - int estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1; + estimatedPos = (int) Math.round(stepRegress.infer(targetTimestamp)) - 1; // search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller // than the given timestamp @@ -367,15 +377,18 @@ public class ChunkSuit4CPV { while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { estimatedPos--; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; } } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { estimatedPos++; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; } if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { estimatedPos--; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; } // else equal } // else equal this.endPos = estimatedPos; // note this @@ -413,12 +426,13 @@ public class ChunkSuit4CPV { } else { // to find the closet timestamp equal to or smaller // than the given timestamp - int estimatedPos = -1; + estimatedPos = -1; pageReader.timeBuffer.position(0); pageReader.valueBuffer.position(pageReader.timeBufferLength); while (pageReader.timeBuffer.remaining() > 0) { estimatedPos++; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; long t = pageReader.timeBuffer.getLong(); if (t >= targetTimestamp) { break; @@ -426,6 +440,7 @@ public class ChunkSuit4CPV { } if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; estimatedPos--; } // else equals no need to minus 1 @@ -460,8 +475,9 @@ public class ChunkSuit4CPV { default: throw new IOException("Unsupported data type!"); } - return estimatedPos; } + IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_b_genLP, System.nanoTime() - start); + return estimatedPos; } /** @@ -471,6 +487,8 @@ public class ChunkSuit4CPV { * @return true if exists; false not exist */ public boolean checkIfExist(long targetTimestamp) throws IOException { + long start = System.nanoTime(); + boolean exist; if (TSFileDescriptor.getInstance().getConfig().isUseChunkIndex()) { StepRegress stepRegress = chunkMetadata.getStatistics().getStepRegress(); // infer position starts from 1, so minus 1 here @@ -483,22 +501,25 @@ public class ChunkSuit4CPV { while (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { estimatedPos--; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; } } else if (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { while (pageReader.timeBuffer.getLong(estimatedPos * 8) < targetTimestamp) { estimatedPos++; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; } if (pageReader.timeBuffer.getLong(estimatedPos * 8) > targetTimestamp) { estimatedPos--; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; } // else equal } // else equal // since we have constrained that targetTimestamp must be within the chunk time range // [startTime, endTime], // estimatedPos will not be out of range. - return pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp; + exist = pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp; } else { // search from estimatePos in the timeBuffer to find the closet timestamp equal to or smaller // than the given timestamp @@ -510,6 +531,7 @@ public class ChunkSuit4CPV { if (!flag) { estimatedPos++; IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; } long t = pageReader.timeBuffer.getLong(); if (t >= targetTimestamp) { @@ -520,8 +542,10 @@ public class ChunkSuit4CPV { // since we have constrained that targetTimestamp must be within the chunk time range // [startTime, endTime], // estimatedPos will not be out of range. - return pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp; + exist = pageReader.timeBuffer.getLong(estimatedPos * 8) == targetTimestamp; } + IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_a_verifBPTP, System.nanoTime() - start); + return exist; } public void updateFP(MinMaxInfo point) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java index d9eeaf30b6c..9c17f371e34 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/IOMonitor2.java @@ -19,10 +19,8 @@ package org.apache.iotdb.tsfile.read.common; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class IOMonitor2 { + public enum DataSetType { // dataSet, executor, reader, file NONE, RawQueryDataSetWithoutValueFilter, @@ -33,12 +31,21 @@ public class IOMonitor2 { public enum Operation { DCP_Server_Query_Execute("DCP_Server_Query_Execute"), DCP_Server_Query_Fetch("DCP_Server_Query_Fetch"), - DCP_SeriesScanOperator_hasNext("DCP_SeriesScanOperator_hasNext"), DCP_A_GET_CHUNK_METADATAS("DCP_A_GET_CHUNK_METADATAS"), DCP_B_READ_MEM_CHUNK("DCP_B_READ_MEM_CHUNK"), DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA( "DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA"), - DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS"); + DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS"), + SEARCH_ARRAY_a_verifBPTP("SEARCH_ARRAY_a_verifBPTP"), + SEARCH_ARRAY_b_genFP("SEARCH_ARRAY_b_genFP"), + SEARCH_ARRAY_b_genLP("SEARCH_ARRAY_b_genLP"), + SEARCH_ARRAY_c_genBPTP("SEARCH_ARRAY_c_genBPTP"), + M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS("M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS"), + M4_LSM_MERGE_M4_TIME_SPAN("M4_LSM_MERGE_M4_TIME_SPAN"), + M4_LSM_FP("M4_LSM_FP"), + M4_LSM_LP("M4_LSM_LP"), + M4_LSM_BP("M4_LSM_BP"), + M4_LSM_TP("M4_LSM_TP"); public String getName() { return name; @@ -51,23 +58,14 @@ public class IOMonitor2 { } } - /** - * [Raw data query metrics] - server端execute&fetch的次数和耗时 - - * SeriesScanOperator.hasNext的次数和耗时（问题：root.hasNext执行次数等于decode - * page次数，但是SeriesScanOperator.hasNext次数大于那个次数？破案了：是root.isFinished->SeriesScanOperator.isFinished里面调用了hasNext，和root.hasNext里调用SeriesScanOperator.hasNext的次数加起来就对了。） - * - A：从磁盘解读chunk元数据的次数和耗时 - B：从磁盘加载chunk数据的次数和耗时 - C：解压缩所有page构造pageReaderList的次数和耗时 - - * D：解码page并遍历点的次数和耗时 （遍历点数不用统计，因为一定是全量点） - */ - public static int DCP_Server_Query_Execute_count = 0; + public static DataSetType dataSetType = DataSetType.NONE; + public static int DCP_Server_Query_Execute_count = 0; public static long DCP_Server_Query_Execute_ns = 0; public static int DCP_Server_Query_Fetch_count = 0; public static long DCP_Server_Query_Fetch_ns = 0; - public static int DCP_SeriesScanOperator_hasNext_count = 0; - public static long DCP_SeriesScanOperator_hasNext_ns = 0; - public static int DCP_A_GET_CHUNK_METADATAS_count = 0; public static long DCP_A_GET_CHUNK_METADATAS_ns = 0; @@ -77,25 +75,49 @@ public class IOMonitor2 { public static int DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0; public static long DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0; + public static long DCP_D_traversedPointNum = 0; public static int DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0; public static long DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0; - public static long DCP_D_traversedPointNum = 0; + public static int SEARCH_ARRAY_a_verifBPTP_count = 0; + public static long SEARCH_ARRAY_a_verifBPTP_ns = 0; - public static DataSetType dataSetType = DataSetType.NONE; + public static int SEARCH_ARRAY_b_genFP_count = 0; + public static long SEARCH_ARRAY_b_genFP_ns = 0; + + public static int SEARCH_ARRAY_b_genLP_count = 0; + public static long SEARCH_ARRAY_b_genLP_ns = 0; + + public static int SEARCH_ARRAY_c_genBPTP_count = 0; + public static long SEARCH_ARRAY_c_genBPTP_ns = 0; + + public static int M4_LSM_init_loadAllChunkMetadatas_count = 0; + public static long M4_LSM_init_loadAllChunkMetadatas_ns = 0; - private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("IOMonitor2"); + public static int M4_LSM_merge_M4_time_span_count = 0; + public static long M4_LSM_merge_M4_time_span_ns = 0; + + public static int M4_LSM_FP_count = 0; + public static long M4_LSM_FP_ns = 0; + + public static int M4_LSM_LP_count = 0; + public static long M4_LSM_LP_ns = 0; + + public static int M4_LSM_BP_count = 0; + public static long M4_LSM_BP_ns = 0; + + public static int M4_LSM_TP_count = 0; + public static long M4_LSM_TP_ns = 0; private static void reset() { + dataSetType = DataSetType.NONE; + DCP_Server_Query_Execute_count = 0; DCP_Server_Query_Execute_ns = 0; DCP_Server_Query_Fetch_count = 0; DCP_Server_Query_Fetch_ns = 0; - DCP_SeriesScanOperator_hasNext_count = 0; - DCP_SeriesScanOperator_hasNext_ns = 0; - DCP_A_GET_CHUNK_METADATAS_count = 0; DCP_A_GET_CHUNK_METADATAS_ns = 0; @@ -105,17 +127,39 @@ public class IOMonitor2 { DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count = 0; DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns = 0; + DCP_D_traversedPointNum = 0; DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count = 0; DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns = 0; - DCP_D_traversedPointNum = 0; + SEARCH_ARRAY_a_verifBPTP_count = 0; + SEARCH_ARRAY_a_verifBPTP_ns = 0; + SEARCH_ARRAY_b_genFP_count = 0; + SEARCH_ARRAY_b_genFP_ns = 0; + SEARCH_ARRAY_b_genLP_count = 0; + SEARCH_ARRAY_b_genLP_ns = 0; + SEARCH_ARRAY_c_genBPTP_count = 0; + SEARCH_ARRAY_c_genBPTP_ns = 0; - dataSetType = DataSetType.NONE; + M4_LSM_init_loadAllChunkMetadatas_count = 0; + M4_LSM_init_loadAllChunkMetadatas_ns = 0; + + M4_LSM_merge_M4_time_span_count = 0; + M4_LSM_merge_M4_time_span_ns = 0; + + M4_LSM_FP_count = 0; + M4_LSM_FP_ns = 0; + + M4_LSM_LP_count = 0; + M4_LSM_LP_ns = 0; + + M4_LSM_BP_count = 0; + M4_LSM_BP_ns = 0; + + M4_LSM_TP_count = 0; + M4_LSM_TP_ns = 0; } public static void addMeasure(Operation operation, long elapsedTimeInNanosecond) { - // TODO tmp for debug - // DEBUG_LOGGER.info(operation.getName() + ": " + elapsedTimeInNanosecond + " ns"); switch (operation) { case DCP_Server_Query_Execute: DCP_Server_Query_Execute_count++; @@ -125,10 +169,6 @@ public class IOMonitor2 { DCP_Server_Query_Fetch_count++; DCP_Server_Query_Fetch_ns += elapsedTimeInNanosecond; break; - case DCP_SeriesScanOperator_hasNext: - DCP_SeriesScanOperator_hasNext_count++; - DCP_SeriesScanOperator_hasNext_ns += elapsedTimeInNanosecond; - break; case DCP_A_GET_CHUNK_METADATAS: DCP_A_GET_CHUNK_METADATAS_count++; DCP_A_GET_CHUNK_METADATAS_ns += elapsedTimeInNanosecond; @@ -145,6 +185,46 @@ public class IOMonitor2 { DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count++; DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns += elapsedTimeInNanosecond; break; + case SEARCH_ARRAY_a_verifBPTP: + SEARCH_ARRAY_a_verifBPTP_count++; + SEARCH_ARRAY_a_verifBPTP_ns += elapsedTimeInNanosecond; + break; + case SEARCH_ARRAY_b_genFP: + SEARCH_ARRAY_b_genFP_count++; + SEARCH_ARRAY_b_genFP_ns += elapsedTimeInNanosecond; + break; + case SEARCH_ARRAY_b_genLP: + SEARCH_ARRAY_b_genLP_count++; + SEARCH_ARRAY_b_genLP_ns += elapsedTimeInNanosecond; + break; + case SEARCH_ARRAY_c_genBPTP: + SEARCH_ARRAY_c_genBPTP_count++; + SEARCH_ARRAY_c_genBPTP_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS: + M4_LSM_init_loadAllChunkMetadatas_count++; + M4_LSM_init_loadAllChunkMetadatas_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_MERGE_M4_TIME_SPAN: + M4_LSM_merge_M4_time_span_count++; + M4_LSM_merge_M4_time_span_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_FP: + M4_LSM_FP_count++; + M4_LSM_FP_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_LP: + M4_LSM_LP_count++; + M4_LSM_LP_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_BP: + M4_LSM_BP_count++; + M4_LSM_BP_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_TP: + M4_LSM_TP_count++; + M4_LSM_TP_ns += elapsedTimeInNanosecond; + break; default: System.out.println("not supported operation type"); // this will not happen break; @@ -161,10 +241,6 @@ public class IOMonitor2 { DCP_Server_Query_Fetch_count += count; DCP_Server_Query_Fetch_ns += elapsedTimeInNanosecond; break; - case DCP_SeriesScanOperator_hasNext: - DCP_SeriesScanOperator_hasNext_count += count; - DCP_SeriesScanOperator_hasNext_ns += elapsedTimeInNanosecond; - break; case DCP_A_GET_CHUNK_METADATAS: DCP_A_GET_CHUNK_METADATAS_count += count; DCP_A_GET_CHUNK_METADATAS_ns += elapsedTimeInNanosecond; @@ -181,6 +257,46 @@ public class IOMonitor2 { DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count += count; DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns += elapsedTimeInNanosecond; break; + case SEARCH_ARRAY_a_verifBPTP: + SEARCH_ARRAY_a_verifBPTP_count += count; + SEARCH_ARRAY_a_verifBPTP_ns += elapsedTimeInNanosecond; + break; + case SEARCH_ARRAY_b_genFP: + SEARCH_ARRAY_b_genFP_count += count; + SEARCH_ARRAY_b_genFP_ns += elapsedTimeInNanosecond; + break; + case SEARCH_ARRAY_b_genLP: + SEARCH_ARRAY_b_genLP_count += count; + SEARCH_ARRAY_b_genLP_ns += elapsedTimeInNanosecond; + break; + case SEARCH_ARRAY_c_genBPTP: + SEARCH_ARRAY_c_genBPTP_count += count; + SEARCH_ARRAY_c_genBPTP_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS: + M4_LSM_init_loadAllChunkMetadatas_count += count; + M4_LSM_init_loadAllChunkMetadatas_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_MERGE_M4_TIME_SPAN: + M4_LSM_merge_M4_time_span_count += count; + M4_LSM_merge_M4_time_span_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_FP: + M4_LSM_FP_count += count; + M4_LSM_FP_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_LP: + M4_LSM_LP_count += count; + M4_LSM_LP_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_BP: + M4_LSM_BP_count += count; + M4_LSM_BP_ns += elapsedTimeInNanosecond; + break; + case M4_LSM_TP: + M4_LSM_TP_count += count; + M4_LSM_TP_ns += elapsedTimeInNanosecond; + break; default: System.out.println("not supported operation type"); // this will not happen break; @@ -202,22 +318,59 @@ public class IOMonitor2 { .append(DCP_Server_Query_Fetch_ns) .append("\n"); stringBuilder - .append("SeriesScanOperator_hasNext_ns") + .append("M4_LSM_init_loadAllChunkMetadatas_ns") + .append(",") + .append(M4_LSM_init_loadAllChunkMetadatas_ns) + .append("\n"); + stringBuilder + .append("M4_LSM_merge_M4_time_span_ns") + .append(",") + .append(M4_LSM_merge_M4_time_span_ns) + .append("\n"); + stringBuilder.append("M4_LSM_FP_ns").append(",").append(M4_LSM_FP_ns).append("\n"); + stringBuilder.append("M4_LSM_LP_ns").append(",").append(M4_LSM_LP_ns).append("\n"); + stringBuilder.append("M4_LSM_BP_ns").append(",").append(M4_LSM_BP_ns).append("\n"); + stringBuilder.append("M4_LSM_TP_ns").append(",").append(M4_LSM_TP_ns).append("\n"); + stringBuilder + .append("DCP_A_GET_CHUNK_METADATAS_ns") + .append(",") + .append(DCP_A_GET_CHUNK_METADATAS_ns) + .append("\n"); + stringBuilder + .append("DCP_B_READ_MEM_CHUNK_ns") .append(",") - .append(DCP_SeriesScanOperator_hasNext_ns) + .append(DCP_B_READ_MEM_CHUNK_ns) .append("\n"); - stringBuilder.append("A_ns").append(",").append(DCP_A_GET_CHUNK_METADATAS_ns).append("\n"); - stringBuilder.append("B_ns").append(",").append(DCP_B_READ_MEM_CHUNK_ns).append("\n"); stringBuilder - .append("C_ns") + .append("DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns") .append(",") .append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_ns) .append("\n"); stringBuilder - .append("D_ns") + .append("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns") .append(",") .append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_ns) .append("\n"); + stringBuilder + .append("SEARCH_ARRAY_a_verifBPTP_ns") + .append(",") + .append(SEARCH_ARRAY_a_verifBPTP_ns) + .append("\n"); + stringBuilder + .append("SEARCH_ARRAY_b_genFP_ns") + .append(",") + .append(SEARCH_ARRAY_b_genFP_ns) + .append("\n"); + stringBuilder + .append("SEARCH_ARRAY_b_genLP_ns") + .append(",") + .append(SEARCH_ARRAY_b_genLP_ns) + .append("\n"); + stringBuilder + .append("SEARCH_ARRAY_c_genBPTP_ns") + .append(",") + .append(SEARCH_ARRAY_c_genBPTP_ns) + .append("\n"); stringBuilder .append("Server_Query_Execute_cnt") @@ -230,22 +383,59 @@ public class IOMonitor2 { .append(DCP_Server_Query_Fetch_count) .append("\n"); stringBuilder - .append("SeriesScanOperator_hasNext_cnt") + .append("M4_LSM_init_loadAllChunkMetadatas_cnt") + .append(",") + .append(M4_LSM_init_loadAllChunkMetadatas_count) + .append("\n"); + stringBuilder + .append("M4_LSM_merge_M4_time_span_cnt") .append(",") - .append(DCP_SeriesScanOperator_hasNext_count) + .append(M4_LSM_merge_M4_time_span_count) .append("\n"); - stringBuilder.append("A_cnt").append(",").append(DCP_A_GET_CHUNK_METADATAS_count).append("\n"); - stringBuilder.append("B_cnt").append(",").append(DCP_B_READ_MEM_CHUNK_count).append("\n"); + stringBuilder.append("M4_LSM_FP_cnt").append(",").append(M4_LSM_FP_count).append("\n"); + stringBuilder.append("M4_LSM_LP_cnt").append(",").append(M4_LSM_LP_count).append("\n"); + stringBuilder.append("M4_LSM_BP_cnt").append(",").append(M4_LSM_BP_count).append("\n"); + stringBuilder.append("M4_LSM_TP_cnt").append(",").append(M4_LSM_TP_count).append("\n"); stringBuilder - .append("C_cnt") + .append("DCP_A_GET_CHUNK_METADATAS_cnt") + .append(",") + .append(DCP_A_GET_CHUNK_METADATAS_count) + .append("\n"); + stringBuilder + .append("DCP_B_READ_MEM_CHUNK_cnt") + .append(",") + .append(DCP_B_READ_MEM_CHUNK_count) + .append("\n"); + stringBuilder + .append("DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_cnt") .append(",") .append(DCP_C_DESERIALIZE_PAGEHEADER_DECOMPRESS_PAGEDATA_count) .append("\n"); stringBuilder - .append("D_cnt") + .append("DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_cnt") .append(",") .append(DCP_D_DECODE_PAGEDATA_TRAVERSE_POINTS_count) .append("\n"); + stringBuilder + .append("SEARCH_ARRAY_a_verifBPTP_cnt") + .append(",") + .append(SEARCH_ARRAY_a_verifBPTP_count) + .append("\n"); + stringBuilder + .append("SEARCH_ARRAY_b_genFP_cnt") + .append(",") + .append(SEARCH_ARRAY_b_genFP_count) + .append("\n"); + stringBuilder + .append("SEARCH_ARRAY_b_genLP_cnt") + .append(",") + .append(SEARCH_ARRAY_b_genLP_count) + .append("\n"); + stringBuilder + .append("SEARCH_ARRAY_c_genBPTP_cnt") + .append(",") + .append(SEARCH_ARRAY_c_genBPTP_count) + .append("\n"); stringBuilder .append("DCP_D_traversedPointNum") .append(",") diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java index 420a32735ac..cc667ab3844 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java @@ -168,6 +168,7 @@ public class PageReader implements IPageReader { } public void updateBPTP(ChunkSuit4CPV chunkSuit4CPV) { + long start = System.nanoTime(); deleteCursor = 0; // TODO DEBUG Statistics statistics = null; switch (dataType) { @@ -187,6 +188,7 @@ public class PageReader implements IPageReader { int count = 0; // update here, not in statistics for (int pos = chunkSuit4CPV.startPos; pos <= chunkSuit4CPV.endPos; pos++) { IOMonitor.incPointsTravered(); + IOMonitor2.DCP_D_traversedPointNum++; long timestamp = timeBuffer.getLong(pos * 8); switch (dataType) { case INT64: @@ -229,6 +231,7 @@ public class PageReader implements IPageReader { } else { chunkSuit4CPV.statistics.setCount(0); // otherwise count won't be zero } + IOMonitor2.addMeasure(Operation.SEARCH_ARRAY_c_genBPTP, System.nanoTime() - start); } /** @return the returned BatchData may be empty, but never be null */
