This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 455b4cf4 Implement chunk reader without statistics (#133)
455b4cf4 is described below
commit 455b4cf4b31203ac68718da5b9dcdc8af6dfb896
Author: Caideyipi <[email protected]>
AuthorDate: Mon Jun 24 12:20:57 2024 +0800
Implement chunk reader without statistics (#133)
---
.../read/reader/chunk/AlignedChunkReader.java | 10 +-
.../chunk/AlignedChunkReaderWithoutStatistics.java | 45 ++++++++
.../tsfile/read/reader/chunk/ChunkReader.java | 2 +-
.../reader/chunk/ChunkReaderWithoutStatistics.java | 37 +++++++
.../AlignedChunkReaderWithoutStatisticsTest.java | 121 +++++++++++++++++++++
.../reader/ChunkReaderWithoutStatisticsTest.java | 110 +++++++++++++++++++
6 files changed, 321 insertions(+), 4 deletions(-)
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java
index c967a298..7191d853 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReader.java
@@ -133,7 +133,7 @@ public class AlignedChunkReader extends AbstractChunkReader
{
}
}
- if (isAllNull || timePageHeader.getEndTime() < readStopTime) {
+ if (isAllNull || isEarlierThanReadStopTime(timePageHeader)) {
// when there is only one page in the chunk, the page statistic is the
same as the chunk, so
// we needn't filter the page again
skipCurrentPage(timePageHeader, valuePageHeaderList);
@@ -159,13 +159,17 @@ public class AlignedChunkReader extends
AbstractChunkReader {
}
}
- if (isAllNull || timePageHeader.getEndTime() < readStopTime ||
pageCanSkip(timePageHeader)) {
+ if (isAllNull || isEarlierThanReadStopTime(timePageHeader) ||
pageCanSkip(timePageHeader)) {
skipCurrentPage(timePageHeader, valuePageHeaderList);
return null;
}
return constructAlignedPageReader(timePageHeader, valuePageHeaderList);
}
+ protected boolean isEarlierThanReadStopTime(final PageHeader timePageHeader)
{
+ return timePageHeader.getEndTime() < readStopTime;
+ }
+
private boolean pageCanSkip(PageHeader pageHeader) {
return queryFilter != null
&& !queryFilter.satisfyStartEndTime(pageHeader.getStartTime(),
pageHeader.getEndTime());
@@ -253,7 +257,7 @@ public class AlignedChunkReader extends AbstractChunkReader
{
return alignedPageReader;
}
- private boolean pageDeleted(PageHeader pageHeader, List<TimeRange>
deleteIntervals) {
+ protected boolean pageDeleted(PageHeader pageHeader, List<TimeRange>
deleteIntervals) {
if (pageHeader.getEndTime() < readStopTime) {
return true;
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReaderWithoutStatistics.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReaderWithoutStatistics.java
new file mode 100644
index 00000000..b9fe8b72
--- /dev/null
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AlignedChunkReaderWithoutStatistics.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.reader.chunk;
+
+import org.apache.tsfile.file.header.PageHeader;
+import org.apache.tsfile.read.common.Chunk;
+import org.apache.tsfile.read.common.TimeRange;
+
+import java.io.IOException;
+import java.util.List;
+
+public class AlignedChunkReaderWithoutStatistics extends AlignedChunkReader {
+ public AlignedChunkReaderWithoutStatistics(
+ final Chunk timeChunk, final List<Chunk> valueChunkList) throws
IOException {
+ super(timeChunk, valueChunkList);
+ }
+
+ @Override
+ protected boolean isEarlierThanReadStopTime(final PageHeader timePageHeader)
{
+ return false;
+ }
+
+ @Override
+ protected boolean pageDeleted(
+ final PageHeader pageHeader, final List<TimeRange> deleteIntervals) {
+ return false;
+ }
+}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java
index a85683b1..dc5af465 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java
@@ -99,7 +99,7 @@ public class ChunkReader extends AbstractChunkReader {
&& !queryFilter.satisfyStartEndTime(pageHeader.getStartTime(),
pageHeader.getEndTime());
}
- private boolean pageDeleted(PageHeader pageHeader) {
+ protected boolean pageDeleted(PageHeader pageHeader) {
if (readStopTime > pageHeader.getEndTime()) {
// used for chunk reader by timestamp
return true;
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReaderWithoutStatistics.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReaderWithoutStatistics.java
new file mode 100644
index 00000000..e960c89b
--- /dev/null
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReaderWithoutStatistics.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.reader.chunk;
+
+import org.apache.tsfile.file.header.PageHeader;
+import org.apache.tsfile.read.common.Chunk;
+
+import java.io.IOException;
+
+public class ChunkReaderWithoutStatistics extends ChunkReader {
+
+ public ChunkReaderWithoutStatistics(final Chunk chunk) throws IOException {
+ super(chunk);
+ }
+
+ @Override
+ protected boolean pageDeleted(final PageHeader pageHeader) {
+ return false;
+ }
+}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/AlignedChunkReaderWithoutStatisticsTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/AlignedChunkReaderWithoutStatisticsTest.java
new file mode 100644
index 00000000..b3dfc5d5
--- /dev/null
+++
b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/AlignedChunkReaderWithoutStatisticsTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.reader;
+
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.constant.TestConstant;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.ChunkMetadata;
+import org.apache.tsfile.file.metadata.IChunkMetadata;
+import org.apache.tsfile.file.metadata.PlainDeviceID;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Chunk;
+import org.apache.tsfile.read.reader.chunk.AlignedChunkReader;
+import org.apache.tsfile.read.reader.chunk.AlignedChunkReaderWithoutStatistics;
+import org.apache.tsfile.utils.FilePathUtils;
+import org.apache.tsfile.utils.TsFileGeneratorUtils;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
+
+public class AlignedChunkReaderWithoutStatisticsTest {
+ private final String testStorageGroup =
TsFileGeneratorUtils.testStorageGroup;
+ private final File SEQ_DIRS =
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ + "data"
+ + File.separator
+ + "sequence"
+ + File.separator
+ + testStorageGroup
+ + File.separator
+ + "0"
+ + File.separator
+ + "0");
+
+ private File file;
+ private final int oldMaxPointNumInPage =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+ private final int deviceNum = 5;
+ private final int measurementNum = 10;
+
+ @Before
+ public void setUp() throws IOException, WriteProcessException {
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(500);
+ if (!SEQ_DIRS.exists()) {
+ Assert.assertTrue(SEQ_DIRS.mkdirs());
+ }
+ String fileName =
+ System.currentTimeMillis() + FilePathUtils.FILE_NAME_SEPARATOR +
"0-0-0.tsfile";
+ String filePath = SEQ_DIRS.getPath() + File.separator + fileName;
+ file =
+ TsFileGeneratorUtils.generateAlignedTsFile(
+ filePath, deviceNum, measurementNum, 500, 0, 0, 0, 0);
+ }
+
+ @After
+ public void tearDown() {
+
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(oldMaxPointNumInPage);
+ if (file.exists()) {
+ file.delete();
+ }
+ if (SEQ_DIRS.exists()) {
+ SEQ_DIRS.delete();
+ }
+ }
+
+ @Test
+ public void testChunkReaderWithoutStatistics() throws IOException {
+ try (final TsFileSequenceReader tsFileSequenceReader =
+ new TsFileSequenceReader(file.getPath())) {
+ for (int i = 0; i < deviceNum; i++) {
+ final List<AlignedChunkMetadata> chunkMetadataList =
+ tsFileSequenceReader.getAlignedChunkMetadata(
+ new PlainDeviceID(testStorageGroup + PATH_SEPARATOR + "d1000"
+ i));
+ for (final AlignedChunkMetadata chunkMetadata : chunkMetadataList) {
+ Chunk timeChunk =
+ tsFileSequenceReader.readMemChunk(
+ (ChunkMetadata) chunkMetadata.getTimeChunkMetadata());
+ timeChunk = new Chunk(timeChunk.getHeader(), timeChunk.getData());
+ final List<Chunk> valueChunkList = new ArrayList<>();
+ for (final IChunkMetadata valueChunkMetadata :
+ chunkMetadata.getValueChunkMetadataList()) {
+ final Chunk valueChunk =
+ tsFileSequenceReader.readMemChunk((ChunkMetadata)
valueChunkMetadata);
+ valueChunkList.add(new Chunk(valueChunk.getHeader(),
valueChunk.getData()));
+ }
+ final AlignedChunkReader chunkReader =
+ new AlignedChunkReaderWithoutStatistics(timeChunk,
valueChunkList);
+ Assert.assertEquals(1, chunkReader.loadPageReaderList().size());
+ }
+ }
+ }
+ }
+}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/ChunkReaderWithoutStatisticsTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/ChunkReaderWithoutStatisticsTest.java
new file mode 100644
index 00000000..57891b1f
--- /dev/null
+++
b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/ChunkReaderWithoutStatisticsTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.reader;
+
+import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.constant.TestConstant;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.ChunkMetadata;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Chunk;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.read.reader.chunk.ChunkReader;
+import org.apache.tsfile.read.reader.chunk.ChunkReaderWithoutStatistics;
+import org.apache.tsfile.utils.FilePathUtils;
+import org.apache.tsfile.utils.TsFileGeneratorUtils;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
+
+public class ChunkReaderWithoutStatisticsTest {
+ private final String testStorageGroup =
TsFileGeneratorUtils.testStorageGroup;
+ private final File SEQ_DIRS =
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ + "data"
+ + File.separator
+ + "sequence"
+ + File.separator
+ + testStorageGroup
+ + File.separator
+ + "0"
+ + File.separator
+ + "0");
+
+ private File file;
+ private final int oldMaxPointNumInPage =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+ private final int deviceNum = 5;
+ private final int measurementNum = 10;
+
+ @Before
+ public void setUp() throws IOException, WriteProcessException {
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(500);
+ if (!SEQ_DIRS.exists()) {
+ Assert.assertTrue(SEQ_DIRS.mkdirs());
+ }
+ String fileName =
+ System.currentTimeMillis() + FilePathUtils.FILE_NAME_SEPARATOR +
"0-0-0.tsfile";
+ String filePath = SEQ_DIRS.getPath() + File.separator + fileName;
+ file =
+ TsFileGeneratorUtils.generateNonAlignedTsFile(
+ filePath, deviceNum, measurementNum, 500, 0, 0, 0, 0);
+ }
+
+ @After
+ public void tearDown() {
+
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(oldMaxPointNumInPage);
+ if (file.exists()) {
+ file.delete();
+ }
+ if (SEQ_DIRS.exists()) {
+ SEQ_DIRS.delete();
+ }
+ }
+
+ @Test
+ public void testChunkReaderWithoutStatistics() throws IOException {
+ try (final TsFileSequenceReader tsFileSequenceReader =
+ new TsFileSequenceReader(file.getPath())) {
+ for (int i = 0; i < deviceNum; i++) {
+ for (int j = 0; j < measurementNum; j++) {
+ final List<ChunkMetadata> chunkMetadataList =
+ tsFileSequenceReader.getChunkMetadataList(
+ new Path(testStorageGroup + PATH_SEPARATOR + "d" + i, "s" +
j, true));
+ for (final ChunkMetadata chunkMetadata : chunkMetadataList) {
+ final Chunk chunk =
tsFileSequenceReader.readMemChunk(chunkMetadata);
+ final ChunkReader chunkReader =
+ new ChunkReaderWithoutStatistics(new Chunk(chunk.getHeader(),
chunk.getData()));
+ Assert.assertEquals(1, chunkReader.loadPageReaderList().size());
+ }
+ }
+ }
+ }
+ }
+}
