This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch NewTsFile in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 314c38af5197aaf33079761021a28108e655cfbf Author: JackieTien97 <[email protected]> AuthorDate: Wed Nov 25 14:29:30 2020 +0800 change some --- .../iotdb/tsfile/file/header/ChunkHeader.java | 4 ++-- .../iotdb/tsfile/file/metadata/TsFileMetadata.java | 27 ++-------------------- .../metadata/statistics/BooleanStatistics.java | 9 +++++++- .../file/metadata/statistics/DoubleStatistics.java | 19 +++++++++------ .../file/metadata/statistics/FloatStatistics.java | 19 +++++++++------ .../file/metadata/statistics/LongStatistics.java | 27 ++++++++++++++-------- .../iotdb/tsfile/utils/ReadWriteIOUtils.java | 2 +- .../write/writer/ForceAppendTsFileWriter.java | 4 +--- 8 files changed, 55 insertions(+), 56 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java index c318d44..6f9beac 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java @@ -173,7 +173,7 @@ public class ChunkHeader { */ public int serializeTo(OutputStream outputStream) throws IOException { int length = 0; - length += ReadWriteIOUtils.write(MetaMarker.CHUNK_HEADER, outputStream); + length += ReadWriteIOUtils.write(chunkType, outputStream); length += ReadWriteIOUtils.write(measurementID, outputStream); length += ReadWriteIOUtils.write(dataSize, outputStream); length += ReadWriteIOUtils.write(dataType, outputStream); @@ -191,7 +191,7 @@ public class ChunkHeader { */ public int serializeTo(ByteBuffer buffer) { int length = 0; - length += ReadWriteIOUtils.write(MetaMarker.CHUNK_HEADER, buffer); + length += ReadWriteIOUtils.write(chunkType, buffer); length += ReadWriteIOUtils.write(measurementID, buffer); length += ReadWriteIOUtils.write(dataSize, buffer); length += ReadWriteIOUtils.write(dataType, buffer); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java index 6098284..33c5f8d 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java @@ -29,6 +29,7 @@ import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.BloomFilter; import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; /** @@ -36,13 +37,6 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class TsFileMetadata { - // fields below are IoTDB extensions and they does not affect TsFile's - // stand-alone functionality - private int totalChunkNum; - // invalid means a chunk has been rewritten by merge and the chunk's data is in - // another new chunk - private int invalidChunkNum; - // bloom filter private BloomFilter bloomFilter; @@ -66,12 +60,10 @@ public class TsFileMetadata { // metadataIndex fileMetaData.metadataIndex = MetadataIndexNode.deserializeFrom(buffer); - fileMetaData.totalChunkNum = ReadWriteIOUtils.readInt(buffer); - fileMetaData.invalidChunkNum = ReadWriteIOUtils.readInt(buffer); // versionInfo List<Pair<Long, Long>> versionInfo = new ArrayList<>(); - int versionSize = ReadWriteIOUtils.readInt(buffer); + int versionSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer); for (int i = 0; i < versionSize; i++) { long versionPos = ReadWriteIOUtils.readLong(buffer); long version = ReadWriteIOUtils.readLong(buffer); @@ -166,21 +158,6 @@ public class TsFileMetadata { return filter; } - public int getTotalChunkNum() { - return totalChunkNum; - } - - public void setTotalChunkNum(int totalChunkNum) { - this.totalChunkNum = totalChunkNum; - } - - public int getInvalidChunkNum() { - return invalidChunkNum; - } - - public void setInvalidChunkNum(int invalidChunkNum) { - this.invalidChunkNum = invalidChunkNum; - } public long getMetaOffset() { return metaOffset; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java index e24ef46..ddbdf34 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java @@ -200,6 +200,7 @@ public class BooleanStatistics extends Statistics<Boolean> { int byteLen = 0; byteLen += ReadWriteIOUtils.write(firstValue, outputStream); byteLen += ReadWriteIOUtils.write(lastValue, outputStream); + byteLen += ReadWriteIOUtils.write(sumValue, outputStream); return byteLen; } @@ -207,16 +208,22 @@ public class BooleanStatistics extends Statistics<Boolean> { void deserialize(InputStream inputStream) throws IOException { this.firstValue = ReadWriteIOUtils.readBool(inputStream); this.lastValue = ReadWriteIOUtils.readBool(inputStream); + this.sumValue = ReadWriteIOUtils.readLong(inputStream); } @Override void deserialize(ByteBuffer byteBuffer) { this.firstValue = ReadWriteIOUtils.readBool(byteBuffer); this.lastValue = ReadWriteIOUtils.readBool(byteBuffer); + this.sumValue = ReadWriteIOUtils.readLong(byteBuffer); } @Override public String toString() { - return super.toString() + " [firstValue:" + firstValue + ",lastValue:" + lastValue + "]"; + return "BooleanStatistics{" + + "firstValue=" + firstValue + + ", lastValue=" + lastValue + + ", sumValue=" + sumValue + + '}'; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java index ef7293c..aba11ba 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java @@ -18,14 +18,14 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.utils.BytesUtils; -import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.BytesUtils; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; public class DoubleStatistics extends Statistics<Double> { @@ -143,20 +143,25 @@ public class DoubleStatistics extends Statistics<Double> { } @Override - public double getSumValue() { + public double getSumDoubleValue() { return sumValue; } @Override + public long getSumLongValue() { + throw new StatisticsClassException("Double statistics does not support: long sum"); + } + + @Override protected void mergeStatisticsValue(Statistics stats) { DoubleStatistics doubleStats = (DoubleStatistics) stats; if (this.isEmpty) { initializeStats(doubleStats.getMinValue(), doubleStats.getMaxValue(), doubleStats.getFirstValue(), - doubleStats.getLastValue(), doubleStats.getSumValue()); + doubleStats.getLastValue(), doubleStats.sumValue); isEmpty = false; } else { updateStats(doubleStats.getMinValue(), doubleStats.getMaxValue(), doubleStats.getFirstValue(), - doubleStats.getLastValue(), doubleStats.getSumValue(), stats.getStartTime(), stats.getEndTime()); + doubleStats.getLastValue(), doubleStats.sumValue, stats.getStartTime(), stats.getEndTime()); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java index 3080fd7..106ff32 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java @@ -18,14 +18,14 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.utils.BytesUtils; -import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.BytesUtils; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; /** * Statistics for float type. @@ -138,20 +138,25 @@ public class FloatStatistics extends Statistics<Float> { } @Override - public double getSumValue() { + public double getSumDoubleValue() { return sumValue; } @Override + public long getSumLongValue() { + throw new StatisticsClassException("Float statistics does not support: long sum"); + } + + @Override protected void mergeStatisticsValue(Statistics stats) { FloatStatistics floatStats = (FloatStatistics) stats; if (isEmpty) { initializeStats(floatStats.getMinValue(), floatStats.getMaxValue(), floatStats.getFirstValue(), - floatStats.getLastValue(), floatStats.getSumValue()); + floatStats.getLastValue(), floatStats.sumValue); isEmpty = false; } else { updateStats(floatStats.getMinValue(), floatStats.getMaxValue(), floatStats.getFirstValue(), - floatStats.getLastValue(), floatStats.getSumValue(), stats.getStartTime(), stats.getEndTime()); + floatStats.getLastValue(), floatStats.sumValue, stats.getStartTime(), stats.getEndTime()); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java index e4958d3..60cc881 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java @@ -18,14 +18,14 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.utils.BytesUtils; -import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.BytesUtils; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; public class LongStatistics extends Statistics<Long> { @@ -67,7 +67,8 @@ public class LongStatistics extends Statistics<Long> { this.lastValue = lastValue; } - private void updateStats(long minValue, long maxValue, long firstValue, long lastValue, double sumValue, long startTime, long endTime) { + private void updateStats(long minValue, long maxValue, long firstValue, long lastValue, + double sumValue, long startTime, long endTime) { if (minValue < this.minValue) { this.minValue = minValue; } @@ -113,11 +114,16 @@ public class LongStatistics extends Statistics<Long> { } @Override - public double getSumValue() { + public double getSumDoubleValue() { return sumValue; } @Override + public long getSumLongValue() { + throw new StatisticsClassException("Long statistics does not support: long sum"); + } + + @Override void updateStats(long value) { if (isEmpty) { initializeStats(value, value, value, value, value); @@ -154,11 +160,11 @@ public class LongStatistics extends Statistics<Long> { LongStatistics longStats = (LongStatistics) stats; if (isEmpty) { initializeStats(longStats.getMinValue(), longStats.getMaxValue(), longStats.getFirstValue(), - longStats.getLastValue(), longStats.getSumValue()); + longStats.getLastValue(), longStats.sumValue); isEmpty = false; } else { - updateStats(longStats.getMinValue(), longStats.getMaxValue(), longStats.getFirstValue(), longStats.getLastValue(), - longStats.getSumValue(), stats.getStartTime(), stats.getEndTime()); + updateStats(longStats.getMinValue(), longStats.getMaxValue(), longStats.getFirstValue(), + longStats.getLastValue(), longStats.sumValue, stats.getStartTime(), stats.getEndTime()); } } @@ -244,7 +250,8 @@ public class LongStatistics extends Statistics<Long> { @Override public String toString() { - return super.toString() + " [minValue:" + minValue + ",maxValue:" + maxValue + ",firstValue:" + firstValue + + return super.toString() + " [minValue:" + minValue + ",maxValue:" + maxValue + ",firstValue:" + + firstValue + ",lastValue:" + lastValue + ",sumValue:" + sumValue + "]"; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java index 45f00ae..524ee01 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java @@ -769,7 +769,7 @@ public class ReadWriteIOUtils { * read a int + buffer */ public static ByteBuffer readByteBufferWithSelfDescriptionLength(ByteBuffer buffer) { - int byteLength = readInt(buffer); + int byteLength = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer); byte[] bytes = new byte[byteLength]; buffer.get(bytes); ByteBuffer byteBuffer = ByteBuffer.allocate(byteLength); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java index 9dfccf8..6625f68 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java @@ -66,9 +66,7 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter { canWrite = true; versionInfo = tsFileMetadata.getVersionInfo(); - totalChunkNum = tsFileMetadata.getTotalChunkNum(); - invalidChunkNum = tsFileMetadata.getInvalidChunkNum(); - + List<String> devices = reader.getAllDevices(); for (String device : devices) { List<ChunkMetadata> chunkMetadataList = new ArrayList<>();
