This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch iotdb in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 26f00fe8d71a4464a1db37af440815c2022c5f04 Author: Caideyipi <[email protected]> AuthorDate: Mon Jun 24 12:20:57 2024 +0800 Implement chunk reader without statistics (#133) --- .../read/reader/chunk/AlignedChunkReader.java | 10 +- .../chunk/AlignedChunkReaderWithoutStatistics.java | 45 ++++++++ .../tsfile/read/reader/chunk/ChunkReader.java | 2 +- .../reader/chunk/ChunkReaderWithoutStatistics.java | 37 +++++++ .../AlignedChunkReaderWithoutStatisticsTest.java | 121 +++++++++++++++++++++ .../reader/ChunkReaderWithoutStatisticsTest.java | 110 +++++++++++++++++++ 6 files changed, 321 insertions(+), 4 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java index c967a298..7191d853 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java @@ -133,7 +133,7 @@ public class AlignedChunkReader extends AbstractChunkReader { } } - if (isAllNull || timePageHeader.getEndTime() < readStopTime) { + if (isAllNull || isEarlierThanReadStopTime(timePageHeader)) { // when there is only one page in the chunk, the page statistic is the same as the chunk, so // we needn't filter the page again skipCurrentPage(timePageHeader, valuePageHeaderList); @@ -159,13 +159,17 @@ public class AlignedChunkReader extends AbstractChunkReader { } } - if (isAllNull || timePageHeader.getEndTime() < readStopTime || pageCanSkip(timePageHeader)) { + if (isAllNull || isEarlierThanReadStopTime(timePageHeader) || pageCanSkip(timePageHeader)) { skipCurrentPage(timePageHeader, valuePageHeaderList); return null; } return constructAlignedPageReader(timePageHeader, valuePageHeaderList); } + protected boolean isEarlierThanReadStopTime(final PageHeader timePageHeader) { + return timePageHeader.getEndTime() < readStopTime; + } + private boolean pageCanSkip(PageHeader pageHeader) { return queryFilter != null && !queryFilter.satisfyStartEndTime(pageHeader.getStartTime(), pageHeader.getEndTime()); @@ -253,7 +257,7 @@ public class AlignedChunkReader extends AbstractChunkReader { return alignedPageReader; } - private boolean pageDeleted(PageHeader pageHeader, List<TimeRange> deleteIntervals) { + protected boolean pageDeleted(PageHeader pageHeader, List<TimeRange> deleteIntervals) { if (pageHeader.getEndTime() < readStopTime) { return true; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReaderWithoutStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReaderWithoutStatistics.java new file mode 100644 index 00000000..b9fe8b72 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReaderWithoutStatistics.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.reader.chunk; + +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.TimeRange; + +import java.io.IOException; +import java.util.List; + +public class AlignedChunkReaderWithoutStatistics extends AlignedChunkReader { + public AlignedChunkReaderWithoutStatistics( + final Chunk timeChunk, final List<Chunk> valueChunkList) throws IOException { + super(timeChunk, valueChunkList); + } + + @Override + protected boolean isEarlierThanReadStopTime(final PageHeader timePageHeader) { + return false; + } + + @Override + protected boolean pageDeleted( + final PageHeader pageHeader, final List<TimeRange> deleteIntervals) { + return false; + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java index a85683b1..dc5af465 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java @@ -99,7 +99,7 @@ public class ChunkReader extends AbstractChunkReader { && !queryFilter.satisfyStartEndTime(pageHeader.getStartTime(), pageHeader.getEndTime()); } - private boolean pageDeleted(PageHeader pageHeader) { + protected boolean pageDeleted(PageHeader pageHeader) { if (readStopTime > pageHeader.getEndTime()) { // used for chunk reader by timestamp return true; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReaderWithoutStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReaderWithoutStatistics.java new file mode 100644 index 00000000..e960c89b --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReaderWithoutStatistics.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.reader.chunk; + +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.read.common.Chunk; + +import java.io.IOException; + +public class ChunkReaderWithoutStatistics extends ChunkReader { + + public ChunkReaderWithoutStatistics(final Chunk chunk) throws IOException { + super(chunk); + } + + @Override + protected boolean pageDeleted(final PageHeader pageHeader) { + return false; + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/AlignedChunkReaderWithoutStatisticsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/AlignedChunkReaderWithoutStatisticsTest.java new file mode 100644 index 00000000..b3dfc5d5 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/AlignedChunkReaderWithoutStatisticsTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.reader; + +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.constant.TestConstant; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.PlainDeviceID; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.reader.chunk.AlignedChunkReader; +import org.apache.tsfile.read.reader.chunk.AlignedChunkReaderWithoutStatistics; +import org.apache.tsfile.utils.FilePathUtils; +import org.apache.tsfile.utils.TsFileGeneratorUtils; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; + +public class AlignedChunkReaderWithoutStatisticsTest { + private final String testStorageGroup = TsFileGeneratorUtils.testStorageGroup; + private final File SEQ_DIRS = + new File( + TestConstant.BASE_OUTPUT_PATH + + "data" + + File.separator + + "sequence" + + File.separator + + testStorageGroup + + File.separator + + "0" + + File.separator + + "0"); + + private File file; + private final int oldMaxPointNumInPage = + TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage(); + private final int deviceNum = 5; + private final int measurementNum = 10; + + @Before + public void setUp() throws IOException, WriteProcessException { + TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(500); + if (!SEQ_DIRS.exists()) { + Assert.assertTrue(SEQ_DIRS.mkdirs()); + } + String fileName = + System.currentTimeMillis() + FilePathUtils.FILE_NAME_SEPARATOR + "0-0-0.tsfile"; + String filePath = SEQ_DIRS.getPath() + File.separator + fileName; + file = + TsFileGeneratorUtils.generateAlignedTsFile( + filePath, deviceNum, measurementNum, 500, 0, 0, 0, 0); + } + + @After + public void tearDown() { + TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(oldMaxPointNumInPage); + if (file.exists()) { + file.delete(); + } + if (SEQ_DIRS.exists()) { + SEQ_DIRS.delete(); + } + } + + @Test + public void testChunkReaderWithoutStatistics() throws IOException { + try (final TsFileSequenceReader tsFileSequenceReader = + new TsFileSequenceReader(file.getPath())) { + for (int i = 0; i < deviceNum; i++) { + final List<AlignedChunkMetadata> chunkMetadataList = + tsFileSequenceReader.getAlignedChunkMetadata( + new PlainDeviceID(testStorageGroup + PATH_SEPARATOR + "d1000" + i)); + for (final AlignedChunkMetadata chunkMetadata : chunkMetadataList) { + Chunk timeChunk = + tsFileSequenceReader.readMemChunk( + (ChunkMetadata) chunkMetadata.getTimeChunkMetadata()); + timeChunk = new Chunk(timeChunk.getHeader(), timeChunk.getData()); + final List<Chunk> valueChunkList = new ArrayList<>(); + for (final IChunkMetadata valueChunkMetadata : + chunkMetadata.getValueChunkMetadataList()) { + final Chunk valueChunk = + tsFileSequenceReader.readMemChunk((ChunkMetadata) valueChunkMetadata); + valueChunkList.add(new Chunk(valueChunk.getHeader(), valueChunk.getData())); + } + final AlignedChunkReader chunkReader = + new AlignedChunkReaderWithoutStatistics(timeChunk, valueChunkList); + Assert.assertEquals(1, chunkReader.loadPageReaderList().size()); + } + } + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/ChunkReaderWithoutStatisticsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/ChunkReaderWithoutStatisticsTest.java new file mode 100644 index 00000000..57891b1f --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/ChunkReaderWithoutStatisticsTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.reader; + +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.constant.TestConstant; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.reader.chunk.ChunkReader; +import org.apache.tsfile.read.reader.chunk.ChunkReaderWithoutStatistics; +import org.apache.tsfile.utils.FilePathUtils; +import org.apache.tsfile.utils.TsFileGeneratorUtils; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; + +public class ChunkReaderWithoutStatisticsTest { + private final String testStorageGroup = TsFileGeneratorUtils.testStorageGroup; + private final File SEQ_DIRS = + new File( + TestConstant.BASE_OUTPUT_PATH + + "data" + + File.separator + + "sequence" + + File.separator + + testStorageGroup + + File.separator + + "0" + + File.separator + + "0"); + + private File file; + private final int oldMaxPointNumInPage = + TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage(); + private final int deviceNum = 5; + private final int measurementNum = 10; + + @Before + public void setUp() throws IOException, WriteProcessException { + TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(500); + if (!SEQ_DIRS.exists()) { + Assert.assertTrue(SEQ_DIRS.mkdirs()); + } + String fileName = + System.currentTimeMillis() + FilePathUtils.FILE_NAME_SEPARATOR + "0-0-0.tsfile"; + String filePath = SEQ_DIRS.getPath() + File.separator + fileName; + file = + TsFileGeneratorUtils.generateNonAlignedTsFile( + filePath, deviceNum, measurementNum, 500, 0, 0, 0, 0); + } + + @After + public void tearDown() { + TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(oldMaxPointNumInPage); + if (file.exists()) { + file.delete(); + } + if (SEQ_DIRS.exists()) { + SEQ_DIRS.delete(); + } + } + + @Test + public void testChunkReaderWithoutStatistics() throws IOException { + try (final TsFileSequenceReader tsFileSequenceReader = + new TsFileSequenceReader(file.getPath())) { + for (int i = 0; i < deviceNum; i++) { + for (int j = 0; j < measurementNum; j++) { + final List<ChunkMetadata> chunkMetadataList = + tsFileSequenceReader.getChunkMetadataList( + new Path(testStorageGroup + PATH_SEPARATOR + "d" + i, "s" + j, true)); + for (final ChunkMetadata chunkMetadata : chunkMetadataList) { + final Chunk chunk = tsFileSequenceReader.readMemChunk(chunkMetadata); + final ChunkReader chunkReader = + new ChunkReaderWithoutStatistics(new Chunk(chunk.getHeader(), chunk.getData())); + Assert.assertEquals(1, chunkReader.loadPageReaderList().size()); + } + } + } + } + } +}
