This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7d4b568 [ISSUE-2611] An unsequence file that covers too many sequence
file causes OOM query (#2616)
7d4b568 is described below
commit 7d4b5682f55ab91542bb3fb72f34f2c84e2f8d72
Author: Xiangwei Wei <[email protected]>
AuthorDate: Sun Feb 21 14:59:10 2021 +0800
[ISSUE-2611] An unsequence file that covers too many sequence file causes
OOM query (#2616)
---
.../iotdb/db/query/reader/series/SeriesReader.java | 20 +++++++----
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 41 ++++++++++++++++++++++
2 files changed, 55 insertions(+), 6 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index 83ef697..01cb4f1 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -415,7 +415,7 @@ public class SeriesReader {
/*
* try to unpack all overlapped ChunkMetadata to cachedPageReaders
*/
- unpackAllOverlappedChunkMetadataToCachedPageReaders(
+ unpackAllOverlappedChunkMetadataToPageReaders(
orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()),
true);
} else {
/*
@@ -447,7 +447,7 @@ public class SeriesReader {
long endpointTime =
orderUtils.getOverlapCheckTime(firstPageReader.getStatistics());
unpackAllOverlappedTsFilesToTimeSeriesMetadata(endpointTime);
unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(endpointTime,
false);
- unpackAllOverlappedChunkMetadataToCachedPageReaders(endpointTime, false);
+ unpackAllOverlappedChunkMetadataToPageReaders(endpointTime, false);
}
}
@@ -464,6 +464,7 @@ public class SeriesReader {
}
}
}
+
return false;
}
@@ -482,15 +483,22 @@ public class SeriesReader {
> firstPageReader.getStatistics().getStartTime()));
}
- private void unpackAllOverlappedChunkMetadataToCachedPageReaders(long
endpointTime, boolean init)
+ private void unpackAllOverlappedChunkMetadataToPageReaders(long
endpointTime, boolean init)
throws IOException {
if (firstChunkMetadata != null
&& orderUtils.isOverlapped(endpointTime,
firstChunkMetadata.getStatistics())) {
unpackOneChunkMetaData(firstChunkMetadata);
firstChunkMetadata = null;
}
+ // In case unpacking too many sequence chunks
+ boolean hasMeetSeq = false;
while (!cachedChunkMetadata.isEmpty()
&& orderUtils.isOverlapped(endpointTime,
cachedChunkMetadata.peek().getStatistics())) {
+ if (cachedChunkMetadata.peek().isSeq() && hasMeetSeq) {
+ break;
+ } else if (cachedChunkMetadata.peek().isSeq()) {
+ hasMeetSeq = true;
+ }
unpackOneChunkMetaData(cachedChunkMetadata.poll());
}
if (init
@@ -672,7 +680,7 @@ public class SeriesReader {
unpackAllOverlappedTsFilesToTimeSeriesMetadata(timeValuePair.getTimestamp());
unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(
timeValuePair.getTimestamp(), false);
-
unpackAllOverlappedChunkMetadataToCachedPageReaders(timeValuePair.getTimestamp(),
false);
+
unpackAllOverlappedChunkMetadataToPageReaders(timeValuePair.getTimestamp(),
false);
unpackAllOverlappedUnseqPageReadersToMergeReader(timeValuePair.getTimestamp());
// from now, the unsequence reader is all unpacked, so we don't need
to consider it
@@ -755,9 +763,9 @@ public class SeriesReader {
private long updateEndPointTime(long currentPageEndPointTime,
VersionPageReader pageReader) {
if (orderUtils.getAscending()) {
- return Math.max(currentPageEndPointTime,
pageReader.getStatistics().getEndTime());
+ return Math.min(currentPageEndPointTime,
pageReader.getStatistics().getEndTime());
} else {
- return Math.min(currentPageEndPointTime,
pageReader.getStatistics().getStartTime());
+ return Math.max(currentPageEndPointTime,
pageReader.getStatistics().getStartTime());
}
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
index 5d5c46c..1710979 100644
---
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
+++
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
@@ -41,6 +41,7 @@ import org.apache.iotdb.tsfile.read.filter.ValueFilter;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -446,4 +447,44 @@ public class IoTDBSeriesReaderIT {
resultSet.close();
}
}
+
+ /** Test when one un-sequenced file may cover a long time range. */
+ @Test
+ public void queryWithLongRangeUnSeqTest() throws SQLException {
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ // make up data
+ final String INSERT_TEMPLATE = "insert into root.sg.d1(time, s1)
values(%d, %d)";
+ final String FLUSH_CMD = "flush";
+ for (int i = 1; i <= 10; i++) {
+ statement.execute(String.format(INSERT_TEMPLATE, i, i));
+ }
+ statement.execute(FLUSH_CMD);
+ for (int i = 12; i <= 20; i++) {
+ statement.execute(String.format(INSERT_TEMPLATE, i, i));
+ }
+ statement.execute(FLUSH_CMD);
+ for (int i = 21; i <= 110; i++) {
+ statement.execute(String.format(INSERT_TEMPLATE, i, i));
+ if (i % 10 == 0) {
+ statement.execute(FLUSH_CMD);
+ }
+ }
+ // unSeq from here
+ for (int i = 11; i <= 101; i += 10) {
+ statement.execute(String.format(INSERT_TEMPLATE, i, i));
+ }
+ statement.execute(FLUSH_CMD);
+
+ // query from here
+ ResultSet resultSet = statement.executeQuery("select s1 from root.sg.d1
where time > 10");
+ int cnt = 0;
+ while (resultSet.next()) {
+ cnt++;
+ }
+ Assert.assertEquals(100, cnt);
+ }
+ }
}
