This is an automated email from the ASF dual-hosted git repository. xiangweiwei pushed a commit to branch 0.11overlap in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5f374eee8e54e2965ff831411a2f933061264812 Author: Alima777 <[email protected]> AuthorDate: Tue Jun 1 20:37:12 2021 +0800 Fix overlapped data bug --- .../iotdb/db/query/reader/series/SeriesReader.java | 3 + .../db/integration/IoTDBOverlappedPageIT.java | 80 ++++++++++++++++++---- 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java index 46b8c1c..411579c 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java @@ -617,6 +617,9 @@ public class SeriesReader { unpackAllOverlappedChunkMetadataToPageReaders(timeValuePair.getTimestamp(), false); unpackAllOverlappedUnseqPageReadersToMergeReader(timeValuePair.getTimestamp()); + // update if there are unSeqPageReaders are unpacked + timeValuePair = mergeReader.currentTimeValuePair(); + if (firstPageReader != null) { if ((orderUtils.getAscending() && timeValuePair.getTimestamp() > firstPageReader .getStatistics().getEndTime()) || (!orderUtils.getAscending() diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java index 76131ad..c575b17 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBOverlappedPageIT.java @@ -26,6 +26,7 @@ import org.apache.iotdb.jdbc.Config; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -44,31 +45,50 @@ import static org.junit.Assert.fail; public class IoTDBOverlappedPageIT { private static int beforeMaxNumberOfPointsInPage; - private static long beforeMemtableSizeThreshold; + private static boolean enableUnseqCompaction; + + private static String[] dataSet1 = new String[]{ + "SET STORAGE GROUP TO root.sg1", + "CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN", + "INSERT INTO root.sg1.d1(time,s1) values(1, 1)", + "INSERT INTO root.sg1.d1(time,s1) values(10, 10)", + "flush", + "INSERT INTO root.sg1.d1(time,s1) values(20, 20)", + "INSERT INTO root.sg1.d1(time,s1) values(30, 30)", + "flush", + "INSERT INTO root.sg1.d1(time,s1) values(110, 110)", + "flush", + "INSERT INTO root.sg1.d1(time,s1) values(5, 5)", + "INSERT INTO root.sg1.d1(time,s1) values(50, 50)", + "INSERT INTO root.sg1.d1(time,s1) values(100, 100)", + "flush", + "INSERT INTO root.sg1.d1(time,s1) values(15, 15)", + "INSERT INTO root.sg1.d1(time,s1) values(25, 25)", + "flush", + }; @BeforeClass public static void setUp() throws Exception { + Class.forName(Config.JDBC_DRIVER_NAME); EnvironmentUtils.closeStatMonitor(); IoTDBDescriptor.getInstance().getConfig() .setCompactionStrategy(CompactionStrategy.NO_COMPACTION); - beforeMemtableSizeThreshold = IoTDBDescriptor.getInstance().getConfig().getMemtableSizeThreshold(); - IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(1024 * 16); + enableUnseqCompaction = IoTDBDescriptor.getInstance().getConfig().isEnableUnseqCompaction(); + IoTDBDescriptor.getInstance().getConfig().setEnableUnseqCompaction(false); + // max_number_of_points_in_page = 10 beforeMaxNumberOfPointsInPage = TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage(); TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(10); EnvironmentUtils.envSetUp(); - Class.forName(Config.JDBC_DRIVER_NAME); - insertData(); } @AfterClass public static void tearDown() throws Exception { // recovery value - TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(beforeMaxNumberOfPointsInPage); EnvironmentUtils.cleanEnv(); - IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(beforeMemtableSizeThreshold); - IoTDBDescriptor.getInstance().getConfig() - .setCompactionStrategy(CompactionStrategy.LEVEL_COMPACTION); + IoTDBDescriptor.getInstance().getConfig().setCompactionStrategy(CompactionStrategy.LEVEL_COMPACTION); + IoTDBDescriptor.getInstance().getConfig().setEnableUnseqCompaction(enableUnseqCompaction); + IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(beforeMaxNumberOfPointsInPage); } @Test @@ -89,17 +109,51 @@ public class IoTDBOverlappedPageIT { try (Connection connection = DriverManager .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); Statement statement = connection.createStatement()) { + insertData(); + String sql = "select s0 from root.vehicle.d0 where time >= 1 and time <= 110 AND root.vehicle.d0.s0 > 110"; ResultSet resultSet = statement.executeQuery(sql); int cnt = 0; - try { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString("root.vehicle.d0.s0"); + Assert.assertEquals(res[cnt], ans); + cnt++; + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + /** + * Test to check if timeValuePair is updated when there are unSeqPageReaders are unpacked + * in method hasNextOverlappedPage() in SeriesReader.java + */ + @Test + public void selectOverlappedPageTest2() { + String[] res = { + "0,10" + }; + + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + + for (String insertSql : dataSet1) { + statement.execute(insertSql); + } + + boolean hasResultSet = statement.execute("select count(s1) from root.sg1.d1"); + Assert.assertTrue(hasResultSet); + int cnt = 0; + try (ResultSet resultSet = statement.getResultSet()) { while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString("root.vehicle.d0.s0"); + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString("count(root.sg1.d1.s1)"); Assert.assertEquals(res[cnt], ans); cnt++; } - } finally { - resultSet.close(); + Assert.assertEquals(res.length, cnt); } } catch (Exception e) { e.printStackTrace();
