This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch improvedAlign_for_expr in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3c21eeff60977fe67576fc04b56e8a268b0b6b5d Author: jt <[email protected]> AuthorDate: Fri May 27 10:52:17 2022 +0800 add compression statistics --- .../apache/iotdb/tsfile/write/page/PageWriter.java | 36 ++++++++++++++++++++++ .../iotdb/tsfile/write/page/TimePageWriter.java | 7 +++++ .../iotdb/tsfile/write/page/ValuePageWriter.java | 21 +++++++++++++ 3 files changed, 64 insertions(+) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java index 0401908477..097f81edbf 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java @@ -36,6 +36,7 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; +import java.util.concurrent.atomic.AtomicLong; /** * This writer is used to write time-value into a page. It consists of a time encoder, a value @@ -43,6 +44,12 @@ import java.nio.channels.WritableByteChannel; */ public class PageWriter { + public static final AtomicLong timeRawSize = new AtomicLong(); + public static final AtomicLong timeEncodedSize = new AtomicLong(); + public static final AtomicLong valueRawSize = new AtomicLong(); + public static final AtomicLong valueEncodedSize = new AtomicLong(); + public static final AtomicLong compressedSize = new AtomicLong(); + private static final Logger logger = LoggerFactory.getLogger(PageWriter.class); private ICompressor compressor; @@ -82,6 +89,8 @@ public class PageWriter { timeEncoder.encode(time, timeOut); valueEncoder.encode(value, valueOut); statistics.update(time, value); + timeRawSize.addAndGet(Long.BYTES); + valueRawSize.addAndGet(1); } /** write a time value pair into encoder */ @@ -89,6 +98,8 @@ public class PageWriter { timeEncoder.encode(time, timeOut); valueEncoder.encode(value, valueOut); statistics.update(time, value); + timeRawSize.addAndGet(Long.BYTES); + valueRawSize.addAndGet(Short.BYTES); } /** write a time value pair into encoder */ @@ -96,6 +107,8 @@ public class PageWriter { timeEncoder.encode(time, timeOut); valueEncoder.encode(value, valueOut); statistics.update(time, value); + timeRawSize.addAndGet(Long.BYTES); + valueRawSize.addAndGet(Integer.BYTES); } /** write a time value pair into encoder */ @@ -103,6 +116,8 @@ public class PageWriter { timeEncoder.encode(time, timeOut); valueEncoder.encode(value, valueOut); statistics.update(time, value); + timeRawSize.addAndGet(Long.BYTES); + valueRawSize.addAndGet(Long.BYTES); } /** write a time value pair into encoder */ @@ -110,6 +125,8 @@ public class PageWriter { timeEncoder.encode(time, timeOut); valueEncoder.encode(value, valueOut); statistics.update(time, value); + timeRawSize.addAndGet(Long.BYTES); + valueRawSize.addAndGet(Float.BYTES); } /** write a time value pair into encoder */ @@ -117,6 +134,8 @@ public class PageWriter { timeEncoder.encode(time, timeOut); valueEncoder.encode(value, valueOut); statistics.update(time, value); + timeRawSize.addAndGet(Long.BYTES); + valueRawSize.addAndGet(Double.BYTES); } /** write a time value pair into encoder */ @@ -124,6 +143,8 @@ public class PageWriter { timeEncoder.encode(time, timeOut); valueEncoder.encode(value, valueOut); statistics.update(time, value); + timeRawSize.addAndGet(Long.BYTES); + valueRawSize.addAndGet(value.getLength()); } /** write time series into encoder */ @@ -132,6 +153,8 @@ public class PageWriter { timeEncoder.encode(timestamps[i], timeOut); valueEncoder.encode(values[i], valueOut); } + timeRawSize.addAndGet(Long.BYTES * batchSize); + valueRawSize.addAndGet(batchSize); statistics.update(timestamps, values, batchSize); } @@ -141,6 +164,8 @@ public class PageWriter { timeEncoder.encode(timestamps[i], timeOut); valueEncoder.encode(values[i], valueOut); } + timeRawSize.addAndGet(Long.BYTES * batchSize); + valueRawSize.addAndGet(Integer.BYTES * batchSize); statistics.update(timestamps, values, batchSize); } @@ -150,6 +175,8 @@ public class PageWriter { timeEncoder.encode(timestamps[i], timeOut); valueEncoder.encode(values[i], valueOut); } + timeRawSize.addAndGet(Long.BYTES * batchSize); + valueRawSize.addAndGet(Long.BYTES * batchSize); statistics.update(timestamps, values, batchSize); } @@ -159,6 +186,8 @@ public class PageWriter { timeEncoder.encode(timestamps[i], timeOut); valueEncoder.encode(values[i], valueOut); } + timeRawSize.addAndGet(Long.BYTES * batchSize); + valueRawSize.addAndGet(Float.BYTES * batchSize); statistics.update(timestamps, values, batchSize); } @@ -168,6 +197,8 @@ public class PageWriter { timeEncoder.encode(timestamps[i], timeOut); valueEncoder.encode(values[i], valueOut); } + timeRawSize.addAndGet(Long.BYTES * batchSize); + valueRawSize.addAndGet(Double.BYTES * batchSize); statistics.update(timestamps, values, batchSize); } @@ -176,6 +207,8 @@ public class PageWriter { for (int i = 0; i < batchSize; i++) { timeEncoder.encode(timestamps[i], timeOut); valueEncoder.encode(values[i], valueOut); + timeRawSize.addAndGet(Long.BYTES); + valueRawSize.addAndGet(values[i].getLength()); } statistics.update(timestamps, values, batchSize); } @@ -199,6 +232,8 @@ public class PageWriter { buffer.put(timeOut.getBuf(), 0, timeOut.size()); buffer.put(valueOut.getBuf(), 0, valueOut.size()); buffer.flip(); + timeEncodedSize.addAndGet(timeOut.size()); + valueEncodedSize.addAndGet(valueOut.size()); return buffer; } @@ -227,6 +262,7 @@ public class PageWriter { compressor.compress( pageData.array(), pageData.position(), uncompressedSize, compressedBytes); } + PageWriter.compressedSize.addAndGet(compressedSize); // write the page header to IOWriter int sizeWithoutStatistic = 0; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java index 1c668fc881..1bb1d9bf1d 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; +import java.util.concurrent.atomic.AtomicLong; /** * This writer is used to write time into a page. It consists of a time encoder and respective @@ -40,6 +41,9 @@ import java.nio.channels.WritableByteChannel; public class TimePageWriter { private static final Logger logger = LoggerFactory.getLogger(TimePageWriter.class); + public static final AtomicLong timeRawSize = new AtomicLong(); + public static final AtomicLong timeEncodedSize = new AtomicLong(); + public static final AtomicLong timeCompressedSize = new AtomicLong(); private final ICompressor compressor; @@ -71,6 +75,7 @@ public class TimePageWriter { for (int i = 0; i < batchSize; i++) { timeEncoder.encode(timestamps[i], timeOut); } + timeRawSize.addAndGet(batchSize * Long.BYTES); statistics.update(timestamps, batchSize); } @@ -90,6 +95,7 @@ public class TimePageWriter { ByteBuffer buffer = ByteBuffer.allocate(timeOut.size()); buffer.put(timeOut.getBuf(), 0, timeOut.size()); buffer.flip(); + timeEncodedSize.addAndGet(timeOut.size()); return buffer; } @@ -118,6 +124,7 @@ public class TimePageWriter { compressor.compress( pageData.array(), pageData.position(), uncompressedSize, compressedBytes); } + timeCompressedSize.addAndGet(compressedSize); // write the page header to IOWriter int sizeWithoutStatistic = 0; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java index 988575aa2c..aca4a450ad 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java @@ -35,12 +35,18 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; +import java.util.concurrent.atomic.AtomicLong; /** * This writer is used to write value into a page. It consists of a value encoder and respective * OutputStream. */ public class ValuePageWriter { + + public static final AtomicLong valueRawSize = new AtomicLong(); + public static final AtomicLong valueEncodedSize = new AtomicLong(); + public static final AtomicLong valueCompressedSize = new AtomicLong(); + private static final Logger logger = LoggerFactory.getLogger(ValuePageWriter.class); private final ICompressor compressor; @@ -77,6 +83,7 @@ public class ValuePageWriter { public void write(long time, boolean value, boolean isNull) { setBit(isNull); if (!isNull) { + valueRawSize.addAndGet(1); valueEncoder.encode(value, valueOut); statistics.update(time, value); } @@ -86,6 +93,7 @@ public class ValuePageWriter { public void write(long time, short value, boolean isNull) { setBit(isNull); if (!isNull) { + valueRawSize.addAndGet(Short.BYTES); valueEncoder.encode(value, valueOut); statistics.update(time, value); } @@ -95,6 +103,7 @@ public class ValuePageWriter { public void write(long time, int value, boolean isNull) { setBit(isNull); if (!isNull) { + valueRawSize.addAndGet(Integer.BYTES); valueEncoder.encode(value, valueOut); statistics.update(time, value); } @@ -104,6 +113,7 @@ public class ValuePageWriter { public void write(long time, long value, boolean isNull) { setBit(isNull); if (!isNull) { + valueRawSize.addAndGet(Long.BYTES); valueEncoder.encode(value, valueOut); statistics.update(time, value); } @@ -113,6 +123,7 @@ public class ValuePageWriter { public void write(long time, float value, boolean isNull) { setBit(isNull); if (!isNull) { + valueRawSize.addAndGet(Float.BYTES); valueEncoder.encode(value, valueOut); statistics.update(time, value); } @@ -122,6 +133,7 @@ public class ValuePageWriter { public void write(long time, double value, boolean isNull) { setBit(isNull); if (!isNull) { + valueRawSize.addAndGet(Double.BYTES); valueEncoder.encode(value, valueOut); statistics.update(time, value); } @@ -131,6 +143,7 @@ public class ValuePageWriter { public void write(long time, Binary value, boolean isNull) { setBit(isNull); if (!isNull) { + valueRawSize.addAndGet(value.getLength()); valueEncoder.encode(value, valueOut); statistics.update(time, value); } @@ -152,6 +165,7 @@ public class ValuePageWriter { for (int i = 0; i < batchSize; i++) { valueEncoder.encode(values[i], valueOut); } + valueRawSize.addAndGet(batchSize); statistics.update(timestamps, values, batchSize); } @@ -160,6 +174,7 @@ public class ValuePageWriter { for (int i = 0; i < batchSize; i++) { valueEncoder.encode(values[i], valueOut); } + valueRawSize.addAndGet(Integer.BYTES * batchSize); statistics.update(timestamps, values, batchSize); } @@ -168,6 +183,7 @@ public class ValuePageWriter { for (int i = 0; i < batchSize; i++) { valueEncoder.encode(values[i], valueOut); } + valueRawSize.addAndGet(Long.BYTES * batchSize); statistics.update(timestamps, values, batchSize); } @@ -176,6 +192,7 @@ public class ValuePageWriter { for (int i = 0; i < batchSize; i++) { valueEncoder.encode(values[i], valueOut); } + valueRawSize.addAndGet(Float.BYTES * batchSize); statistics.update(timestamps, values, batchSize); } @@ -184,6 +201,7 @@ public class ValuePageWriter { for (int i = 0; i < batchSize; i++) { valueEncoder.encode(values[i], valueOut); } + valueRawSize.addAndGet(Double.BYTES * batchSize); statistics.update(timestamps, values, batchSize); } @@ -191,6 +209,7 @@ public class ValuePageWriter { public void write(long[] timestamps, Binary[] values, int batchSize) { for (int i = 0; i < batchSize; i++) { valueEncoder.encode(values[i], valueOut); + valueRawSize.addAndGet(values[i].getLength()); } statistics.update(timestamps, values, batchSize); } @@ -211,6 +230,7 @@ public class ValuePageWriter { */ public ByteBuffer getUncompressedBytes() throws IOException { prepareEndWriteOnePage(); + valueEncodedSize.addAndGet(Integer.BYTES + bitmapOut.size() + valueOut.size()); ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES + bitmapOut.size() + valueOut.size()); buffer.putInt(size); buffer.put(bitmapOut.getBuf(), 0, bitmapOut.size()); @@ -251,6 +271,7 @@ public class ValuePageWriter { compressor.compress( pageData.array(), pageData.position(), uncompressedSize, compressedBytes); } + valueCompressedSize.addAndGet(compressedSize); // write the page header to IOWriter int sizeWithoutStatistic = 0;
