This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch feature_read_data_from_unclosed_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to
refs/heads/feature_read_data_from_unclosed_tsfile by this push:
new 990f886 temporary commit
990f886 is described below
commit 990f886a8a5b9991927a923aa542b6cbd549f19b
Author: xiangdong huang <[email protected]>
AuthorDate: Tue Mar 5 22:06:02 2019 +0800
temporary commit
---
.../engine/bufferwrite/BufferWriteProcessor.java | 1 -
.../bufferwrite}/RestorableTsFileIOWriter.java | 2 +-
.../bufferwrite}/RestorableTsFileIOWriterTest.java | 5 +-
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 4 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 8 +
.../apache/iotdb/tsfile/write/TsFileWriter.java | 32 ++-
.../apache/iotdb/tsfile/write/record/TSRecord.java | 3 +-
.../tsfile/write/writer/DefaultTsFileOutput.java | 6 +-
.../writer/NativeRestorableTsFileIOWriter.java | 37 +--
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 7 +
.../writer/NativeRestorableTsFileIOWriterTest.java | 260 +++++++++++++++++++++
11 files changed, 344 insertions(+), 21 deletions(-)
diff --git
a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java
b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java
index ecdcaa3..dc56312 100644
---
a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java
+++
b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java
@@ -53,7 +53,6 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
-import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
similarity index 99%
rename from
tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
rename to
iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
index 83b627a..7331104 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
+++
b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.tsfile.write.writer;
+package org.apache.iotdb.db.engine.bufferwrite;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriterTest.java
similarity index 98%
rename from
tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
rename to
iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriterTest.java
index eadcaa5..e341e5b 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriterTest.java
+++
b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriterTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.tsfile.write.writer;
+package org.apache.iotdb.db.engine.bufferwrite;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -46,12 +46,13 @@ import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+
public class RestorableTsFileIOWriterTest {
private RestorableTsFileIOWriter writer;
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
index 83852e0..b4189fa 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
@@ -113,6 +113,10 @@ public class TSFileDescriptor {
TSFileConfig.pageSizeInByte = Integer
.parseInt(properties
.getProperty("page_size_in_byte",
Integer.toString(TSFileConfig.pageSizeInByte)));
+ if (TSFileConfig.pageSizeInByte > TSFileConfig.groupSizeInByte) {
+ LOGGER.warn("page_size is greater than group size, will set it as the
same with group size");
+ TSFileConfig.pageSizeInByte = TSFileConfig.groupSizeInByte;
+ }
TSFileConfig.maxNumberOfPointsInPage = Integer.parseInt(
properties
.getProperty("max_number_of_points_in_page",
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index d50af27..4460098 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -365,4 +365,12 @@ public class TsFileSequenceReader {
return ReadWriteIOUtils
.readAsPossible(tsFileInput.wrapAsFileChannel(), target, position,
length);
}
+
+ /**
+ *
+ * @param pos the position of the file you want to move to
+ */
+ public void setPosition(long pos) throws IOException {
+ tsFileInput.position(pos);
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
index 0a291db..cb7194c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
@@ -22,12 +22,12 @@ import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.exception.write.NoMeasurementException;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
+import org.apache.iotdb.tsfile.file.header.PageHeader;
import org.apache.iotdb.tsfile.write.chunk.ChunkGroupWriterImpl;
import org.apache.iotdb.tsfile.write.chunk.IChunkGroupWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
@@ -91,6 +91,15 @@ public class TsFileWriter {
/**
* init this TsFileWriter.
*
+ * @param fileWriter the io writer of this TsFile
+ */
+ public TsFileWriter(TsFileIOWriter fileWriter) {
+ this(fileWriter, new FileSchema(),
TSFileDescriptor.getInstance().getConfig());
+ }
+
+ /**
+ * init this TsFileWriter.
+ *
* @param file the File to be written by this TsFileWriter
* @param schema the schema of this TsFile
*/
@@ -131,6 +140,11 @@ public class TsFileWriter {
this.schema = schema;
this.pageSize = TSFileConfig.pageSizeInByte;
this.chunkGroupSizeThreshold = TSFileConfig.groupSizeInByte;
+ if (this.pageSize >= chunkGroupSizeThreshold) {
+ LOG.warn(
+ "TsFile's page size {} is greater than chunk group size {}, please
enlarge the chunk group"
+ + " size or decrease page size. ", pageSize,
chunkGroupSizeThreshold);
+ }
}
/**
@@ -290,4 +304,20 @@ public class TsFileWriter {
flushAllChunkGroups();
fileWriter.endFile(this.schema);
}
+
+ /**
+ * this function is only for Test.
+ * @return
+ */
+ public TsFileIOWriter getIOWriter() {
+ return this.fileWriter;
+ }
+
+ /**
+ * this function is only for Test
+ * @throws IOException
+ */
+ public void flushForTest() throws IOException {
+ flushAllChunkGroups();
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/TSRecord.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/TSRecord.java
index b30a483..9ce98a7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/TSRecord.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/TSRecord.java
@@ -64,8 +64,9 @@ public class TSRecord {
*
* @param tuple data point to be added
*/
- public void addTuple(DataPoint tuple) {
+ public TSRecord addTuple(DataPoint tuple) {
this.dataPointList.add(tuple);
+ return this;
}
/**
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/DefaultTsFileOutput.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/DefaultTsFileOutput.java
index 4ea8be8..0ab5e21 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/DefaultTsFileOutput.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/DefaultTsFileOutput.java
@@ -31,12 +31,16 @@ import java.nio.ByteBuffer;
*/
public class DefaultTsFileOutput implements TsFileOutput {
- FileOutputStream outputStream;
+ private FileOutputStream outputStream;
public DefaultTsFileOutput(File file) throws FileNotFoundException {
this.outputStream = new FileOutputStream(file);
}
+ public DefaultTsFileOutput(File file, boolean append) throws
FileNotFoundException {
+ this.outputStream = new FileOutputStream(file, append);
+ }
+
public DefaultTsFileOutput(FileOutputStream outputStream) {
this.outputStream = outputStream;
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableTsFileIOWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableTsFileIOWriter.java
index 19f98a6..35842c3 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableTsFileIOWriter.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableTsFileIOWriter.java
@@ -38,15 +38,22 @@ public class NativeRestorableTsFileIOWriter extends
TsFileIOWriter {
private static final Logger LOGGER = LoggerFactory
.getLogger(NativeRestorableTsFileIOWriter.class);
+ private long truncatedPosition = -1;
- public NativeRestorableTsFileIOWriter(String insertFilePath) throws
IOException {
- super();
- File insertFile = new File(insertFilePath);
+ public long getTruncatedPosition() {
+ return truncatedPosition;
+ }
- if (!insertFile.exists()) {
- this.out = new DefaultTsFileOutput(insertFile);
+ public NativeRestorableTsFileIOWriter(File file) throws IOException {
+ super();
+ long fileSize;
+ if (!file.exists()) {
+ this.out = new DefaultTsFileOutput(file, true);
startFile();
return;
+ } else {
+ fileSize = file.length();
+ this.out = new DefaultTsFileOutput(file, true);
}
//we need to read data to recover TsFileIOWriter.chunkGroupMetaDataList
@@ -68,22 +75,24 @@ public class NativeRestorableTsFileIOWriter extends
TsFileIOWriter {
long versionOfChunkGroup = 0;
boolean newGroup = true;
- TsFileSequenceReader reader = new TsFileSequenceReader(insertFilePath);
- if (reader.fileSize() <= 4) {
- LOGGER.debug("{} does not worth to recover, will create a new one.",
insertFilePath);
+ TsFileSequenceReader reader = new
TsFileSequenceReader(file.getAbsolutePath(), false);
+ if (fileSize <= magicStringBytes.length) {
+ LOGGER.debug("{} only has magic header, does not worth to recover.",
file.getAbsolutePath());
reader.close();
this.out.truncate(0);
startFile();
+ truncatedPosition = magicStringBytes.length;
return;
}
if (reader.readTailMagic().equals(reader.readHeadMagic())) {
- LOGGER.debug("{} is an complete TsFile.", insertFilePath);
+ LOGGER.debug("{} is an complete TsFile.", file.getAbsolutePath());
complete = true;
return;
}
// not a complete file, we will recover it...
- long pos = magicStringBytes.length;
+ truncatedPosition = magicStringBytes.length;
+ reader.setPosition(truncatedPosition);
boolean goon = true;
byte marker;
try {
@@ -141,7 +150,7 @@ public class NativeRestorableTsFileIOWriter extends
TsFileIOWriter {
metadatas.add(currentChunkGroup);
newGroup = true;
//we get a complete chunk group now.
- pos = getPos();
+ truncatedPosition = getPos();
} catch (IOException e) {
goon = false;
}
@@ -154,10 +163,10 @@ public class NativeRestorableTsFileIOWriter extends
TsFileIOWriter {
} catch (IOException e2) {
} finally {
//something wrong or all data is complete. We will discard current
FileMetadata
- out.truncate(pos);
+ LOGGER.info("File {} has {} bytes, and will be truncated from {}.",
+ file.getAbsolutePath(), file.length(), truncatedPosition);
+ out.truncate(truncatedPosition);
}
-
-
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index ae896dc..83616fc 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -321,4 +321,11 @@ public class TsFileIOWriter {
return complete;
}
+ /**
+ * close the inputstream or file channel in force. This is just used for
Testing.
+ */
+ public void forceClose() throws IOException {
+ out.close();
+ }
+
}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableTsFileIOWriterTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableTsFileIOWriterTest.java
new file mode 100644
index 0000000..10bcda0
--- /dev/null
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableTsFileIOWriterTest.java
@@ -0,0 +1,260 @@
+package org.apache.iotdb.tsfile.write.writer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.header.PageHeader;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
+import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.file.metadata.statistics.FloatStatistics;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.FloatDataPoint;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class NativeRestorableTsFileIOWriterTest {
+ private int bkChunkGroupSize;
+
+ private static final String FILE_NAME = "test.ts";
+ @Before
+ public void setUp() throws Exception {
+ bkChunkGroupSize = TSFileConfig.groupSizeInByte;
+ TSFileConfig.groupSizeInByte = 20;
+ TSFileConfig.pageSizeInByte = 5;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ TSFileConfig.groupSizeInByte = bkChunkGroupSize;
+ }
+
+ @Test
+ public void testOnlyHeadMagic() throws Exception {
+ File file = new File(FILE_NAME);
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.getIOWriter().forceClose();
+ NativeRestorableTsFileIOWriter rWriter = new
NativeRestorableTsFileIOWriter(file);
+ writer = new TsFileWriter(rWriter);
+ writer.close();
+ assertEquals(TsFileIOWriter.magicStringBytes.length,
rWriter.getTruncatedPosition());
+ printFile(FILE_NAME);
+
+ assertTrue(file.delete());
+
+ }
+
+ @Test
+ public void testOnlyFirstMask() throws Exception {
+ File file = new File(FILE_NAME);
+ TsFileWriter writer = new TsFileWriter(file);
+ //we have to flush using inner API.
+ writer.getIOWriter().out.write(new byte[] {MetaMarker.CHUNK_HEADER});
+ writer.getIOWriter().forceClose();
+ assertEquals(TsFileIOWriter.magicStringBytes.length + 1, file.length());
+ NativeRestorableTsFileIOWriter rWriter = new
NativeRestorableTsFileIOWriter(file);
+ writer = new TsFileWriter(rWriter);
+ writer.close();
+ printFile(FILE_NAME);
+ assertEquals(TsFileIOWriter.magicStringBytes.length,
rWriter.getTruncatedPosition());
+ assertTrue(file.delete());
+ }
+
+ @Test
+ public void testOnlyOneIncompleteChunkHeader() throws Exception {
+ File file = new File(FILE_NAME);
+ TsFileWriter writer = new TsFileWriter(file);
+
+ ChunkHeader header = new ChunkHeader("s1", 100, TSDataType.FLOAT,
CompressionType.SNAPPY,
+ TSEncoding.PLAIN, 5);
+ ByteBuffer buffer = ByteBuffer.allocate(header.getSerializedSize());
+ header.serializeTo(buffer);
+ buffer.flip();
+ byte[] data = new byte[3];
+ buffer.get(data, 0, 3);
+ writer.getIOWriter().out.write(data);
+ writer.getIOWriter().forceClose();
+ NativeRestorableTsFileIOWriter rWriter = new
NativeRestorableTsFileIOWriter(file);
+ writer = new TsFileWriter(rWriter);
+ writer.close();
+ assertEquals(TsFileIOWriter.magicStringBytes.length,
rWriter.getTruncatedPosition());
+ assertTrue(file.delete());
+ }
+
+ @Test
+ public void testOnlyOneChunkHeader() throws Exception {
+ File file = new File(FILE_NAME);
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.getIOWriter()
+ .startFlushChunk(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.PLAIN),
+ CompressionType.SNAPPY, TSDataType.FLOAT,
+ TSEncoding.PLAIN, new FloatStatistics(), 100, 50, 100, 10);
+ writer.getIOWriter().forceClose();
+
+ NativeRestorableTsFileIOWriter rWriter = new
NativeRestorableTsFileIOWriter(file);
+ writer = new TsFileWriter(rWriter);
+ writer.close();
+ assertEquals(TsFileIOWriter.magicStringBytes.length,
rWriter.getTruncatedPosition());
+ assertTrue(file.delete());
+ }
+
+ @Test
+ public void testOnlyOneChunkHeaderAndSomePage() throws Exception {
+
+ }
+
+ @Test
+ public void testOnlyOneCompleteChunk() throws Exception {
+
+ }
+
+ @Test
+ public void testOnlyOneChunkAndMore() throws Exception {
+
+ }
+
+ @Test
+ public void testOnlyOneChunkGroup() throws Exception {
+ File file = new File(FILE_NAME);
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.addMeasurement(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.addMeasurement(new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(new TSRecord(1, "d1").addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(new TSRecord(2, "d1").addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushForTest();
+
+ NativeRestorableTsFileIOWriter rWriter = new
NativeRestorableTsFileIOWriter(file);
+ writer = new TsFileWriter(rWriter);
+ writer.close();
+ assertEquals(TsFileIOWriter.magicStringBytes.length,
rWriter.getTruncatedPosition());
+ assertTrue(file.delete());
+ }
+
+ @Test
+ public void testOneChunkGroupAndMore() throws Exception {
+
+ }
+
+ @Test
+ public void testNoSeperatorMask() throws Exception {
+
+ }
+
+ @Test
+ public void testHavingSeperatorMask() throws Exception {
+
+ }
+
+ @Test
+ public void testHavingSomeFileMetadata() throws Exception {
+
+ }
+
+ @Test
+ public void testOpenCompleteFile() throws Exception {
+
+ }
+
+ public void printFile(String fileName) throws IOException {
+ TsFileSequenceReader reader = new TsFileSequenceReader(fileName);
+ System.out.println("file length: " + new File(fileName).length());
+ System.out.println("file magic head: " + reader.readHeadMagic());
+ System.out.println("file magic tail: " + reader.readTailMagic());
+ System.out.println("Level 1 metadata position: " +
reader.getFileMetadataPos());
+ System.out.println("Level 1 metadata size: " +
reader.getFileMetadataPos());
+ TsFileMetaData metaData = reader.readFileMetadata();
+ // Sequential reading of one ChunkGroup now follows this order:
+ // first SeriesChunks (headers and data) in one ChunkGroup, then the
CHUNK_GROUP_FOOTER
+ // Because we do not know how many chunks a ChunkGroup may have, we should
read one byte (the marker) ahead and
+ // judge accordingly.
+ System.out.println("[Chunk Group]");
+ System.out.println("position: " + reader.position());
+ byte marker;
+ while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) {
+ switch (marker) {
+ case MetaMarker.CHUNK_HEADER:
+ System.out.println("\t[Chunk]");
+ System.out.println("\tposition: " + reader.position());
+ ChunkHeader header = reader.readChunkHeader();
+ System.out.println("\tMeasurement: " + header.getMeasurementID());
+ Decoder defaultTimeDecoder = Decoder.getDecoderByType(
+
TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().timeSeriesEncoder),
+ TSDataType.INT64);
+ Decoder valueDecoder = Decoder
+ .getDecoderByType(header.getEncodingType(),
header.getDataType());
+ for (int j = 0; j < header.getNumOfPages(); j++) {
+ System.out.println("\t\t[Page]\n \t\tPage head position: " +
reader.position());
+ PageHeader pageHeader =
reader.readPageHeader(header.getDataType());
+ System.out.println("\t\tPage data position: " + reader.position());
+ System.out.println("\t\tpoints in the page: " +
pageHeader.getNumOfValues());
+ ByteBuffer pageData = reader.readPage(pageHeader,
header.getCompressionType());
+ System.out
+ .println("\t\tUncompressed page data size: " +
pageHeader.getUncompressedSize());
+ PageReader reader1 = new PageReader(pageData,
header.getDataType(), valueDecoder,
+ defaultTimeDecoder);
+ while (reader1.hasNextBatch()) {
+ BatchData batchData = reader1.nextBatch();
+ while (batchData.hasNext()) {
+ System.out.println(
+ "\t\t\ttime, value: " + batchData.currentTime() + ", " +
batchData
+ .currentValue());
+ batchData.next();
+ }
+ }
+ }
+ break;
+ case MetaMarker.CHUNK_GROUP_FOOTER:
+ System.out.println("Chunk Group Footer position: " +
reader.position());
+ ChunkGroupFooter chunkGroupFooter = reader.readChunkGroupFooter();
+ System.out.println("device: " + chunkGroupFooter.getDeviceID());
+ break;
+ default:
+ MetaMarker.handleUnexpectedMarker(marker);
+ }
+ }
+ System.out.println("[Metadata]");
+ List<TsDeviceMetadataIndex> deviceMetadataIndexList =
metaData.getDeviceMap().values().stream()
+ .sorted((x, y) -> (int) (x.getOffset() -
y.getOffset())).collect(Collectors.toList());
+ for (TsDeviceMetadataIndex index : deviceMetadataIndexList) {
+ TsDeviceMetadata deviceMetadata = reader.readTsDeviceMetaData(index);
+ List<ChunkGroupMetaData> chunkGroupMetaDataList =
deviceMetadata.getChunkGroupMetaDataList();
+ for (ChunkGroupMetaData chunkGroupMetaData : chunkGroupMetaDataList) {
+ System.out.println(String
+ .format("\t[Device]File Offset: %d, Device %s, Number of Chunk
Groups %d",
+ index.getOffset(), chunkGroupMetaData.getDeviceID(),
+ chunkGroupMetaDataList.size()));
+
+ for (ChunkMetaData chunkMetadata :
chunkGroupMetaData.getChunkMetaDataList()) {
+ System.out.println("\t\tMeasurement:" +
chunkMetadata.getMeasurementUid());
+ System.out.println("\t\tFile offset:" +
chunkMetadata.getOffsetOfChunkHeader());
+ }
+ }
+ }
+ reader.close();
+ }
+
+}
\ No newline at end of file
