This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fc4393309cc27e25a9c26fb578c16bdc85006022
Author: Lei Rui <1010953...@qq.com>
AuthorDate: Thu Feb 1 15:48:09 2024 +0800
disable chunk cache;disable pageReader clear;add traverse metric;fix convex
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../groupby/LocalGroupByExecutorTri_ILTS.java | 42 +++++++++++++---------
.../groupby/LocalGroupByExecutorTri_LTTB.java | 12 +++----
.../iotdb/db/integration/tri/MyTest_ILTS.java | 6 ++--
4 files changed, 36 insertions(+), 26 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 613ee79ce97..ee121e18ed6 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -371,7 +371,7 @@ public class IoTDBConfig {
private int maxSelectUnseqFileNumInEachUnseqCompaction = 2000;
/** whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */
- private boolean metaDataCacheEnable = true;
+ private boolean metaDataCacheEnable = false;
/** Memory allocated for timeSeriesMetaData cache in read process */
private long allocateMemoryForTimeSeriesMetaDataCache =
allocateMemoryForRead / 5;
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java
b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java
index 195be79bf0a..7e32ed46de9 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java
@@ -382,6 +382,7 @@ public class LocalGroupByExecutorTri_ILTS implements
GroupByExecutor {
int count = chunkSuit4Tri.chunkMetadata.getStatistics().getCount();
int j;
for (j = 0; j < count; j++) {
+ IOMonitor2.DCP_D_getAllSatisfiedPageData_traversedPointNum++;
long timestamp = pageReader.timeBuffer.getLong(j * 8);
if (timestamp < localCurStartTime) {
continue;
@@ -391,30 +392,37 @@ public class LocalGroupByExecutorTri_ILTS implements
GroupByExecutor {
ByteBuffer valueBuffer = pageReader.valueBuffer;
double v = valueBuffer.getDouble(pageReader.timeBufferLength + j
* 8);
double distance = IOMonitor2.calculateDistance(lt, lv,
timestamp, v, rt, rv);
- if (distance > maxDistance) {
- maxDistance = distance;
+
+ // TODO 下面假装已经有凸包剪枝，先实验看看如果跳过一些点不用遍历有多少加速效果
+ if (CONFIG.isAcc_convex()) {
+ // if (chunkSuit4Tri.chunkMetadata.getEndTime()
< localCurEndTime) {
+ // chunkSuit4Tri.pageReader = null;
+ // }
select_t = timestamp;
select_v = v;
+ break;
- // TODO 下面假装已经有凸包剪枝，先实验看看如果跳过一些点不用遍历有多少加速效果
- if (CONFIG.isAcc_convex()) {
- if (chunkSuit4Tri.chunkMetadata.getEndTime() <
localCurEndTime) {
- chunkSuit4Tri.pageReader = null;
- }
- break;
+ } else {
+ if (distance > maxDistance) {
+ // TODO
是不是因为开启了acc_rect之后，导致这里要遍历的chunk块里没有点的距离可以达到maxDistance
+ // 从而acc_convex不会生效？！
+ maxDistance = distance;
+ select_t = timestamp;
+ select_v = v;
}
}
}
}
- // clear for heap space
- if (j >= count) {
- // 代表这个chunk已经读完了，后面的bucket不会再用到，所以现在就可以清空内存的page
- //
而不是等到下一个bucket的时候再清空，因为有可能currentChunkList里chunks太多，page点同时存在太多，heap space不够
- chunkSuit4Tri.pageReader = null;
- // TODO 但是这样有可能导致下一轮迭代到这个桶的时候又要读一遍这个chunk
- // 但是不这样做的话相当于一轮迭代之后几乎所有的点都加载到内存留着了
- // 还要注意的是如果被rectangle提前剪枝掉了就不会走到这一步，也就是说那个chunk的pageReader可能还留着
- }
+ // // clear for heap space
+ // if (j >= count) {
+ // // 代表这个chunk已经读完了，后面的bucket不会再用到，所以现在就可以清空内存的page
+ // //
而不是等到下一个bucket的时候再清空，因为有可能currentChunkList里chunks太多，page点同时存在太多，heap
+ // space不够
+ // chunkSuit4Tri.pageReader = null;
+ // // TODO 但是这样有可能导致下一轮迭代到这个桶的时候又要读一遍这个chunk
+ // // 但是不这样做的话相当于一轮迭代之后几乎所有的点都加载到内存留着了
+ // //
还要注意的是如果被rectangle提前剪枝掉了就不会走到这一步，也就是说那个chunk的pageReader可能还留着
+ // }
}
// // 记录结果 // TODO debug
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_LTTB.java
b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_LTTB.java
index f29956c7b8d..1c802efdb4c 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_LTTB.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_LTTB.java
@@ -265,12 +265,12 @@ public class LocalGroupByExecutorTri_LTTB implements
GroupByExecutor {
}
}
}
- // clear for heap space
- if (j >= count) {
- // 代表这个chunk已经读完了，后面的bucket不会再用到，所以现在就可以清空内存的page
- //
而不是等到下一个bucket的时候再清空，因为有可能currentChunkList里chunks太多，page点同时存在太多，heap space不够
- chunkSuit4Tri.pageReader = null;
- }
+ // // clear for heap space
+ // if (j >= count) {
+ // // 代表这个chunk已经读完了，后面的bucket不会再用到，所以现在就可以清空内存的page
+ // //
而不是等到下一个bucket的时候再清空，因为有可能currentChunkList里chunks太多，page点同时存在太多，heap space不够
+ // chunkSuit4Tri.pageReader = null;
+ // }
}
// 记录结果
series.append(select_v).append("[").append(select_t).append("]").append(",");
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java
b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java
index 1e8e1f47180..2e7a3331172 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.jdbc.IoTDBStatement;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.junit.After;
@@ -80,7 +81,7 @@ public class MyTest_ILTS {
// config.setNumIterations(4);
config.setAcc_avg(true);
config.setAcc_rectangle(true);
- config.setAcc_convex(true);
+ config.setAcc_convex(false);
config.setAcc_iterRepeat(true);
config.setEnableCPV(false);
@@ -163,7 +164,7 @@ public class MyTest_ILTS {
@Test
public void test1_3() {
prepareData1();
- config.setNumIterations(2);
+ config.setNumIterations(4);
String res = "5.0[1],2.0[40],20.0[62],7.0[102],";
try (Connection connection =
DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/",
"root", "root");
@@ -186,6 +187,7 @@ public class MyTest_ILTS {
Assert.assertEquals(res, ans);
}
}
+ System.out.println(((IoTDBStatement) statement).executeFinish());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
