This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch float_encoder_overflow in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit fc0e9ecfdadc24d63b76aa22b705a8b60bd2ede7 Author: HTHou <[email protected]> AuthorDate: Thu Dec 19 17:04:00 2024 +0800 try one try --- .../tsfile/encoding/decoder/FloatDecoder.java | 32 +++++++++++++++++-- .../tsfile/encoding/encoder/FloatEncoder.java | 36 ++++++++++++++++++++-- .../tsfile/encoding/decoder/FloatDecoderTest.java | 20 ++++++++++++ 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatDecoder.java index 6b019930..f307a4c4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatDecoder.java @@ -31,6 +31,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; /** * Decoder for float or double value using rle or two diff. For more info about encoding pattern, @@ -47,6 +49,9 @@ public class FloatDecoder extends Decoder { /** flag that indicates whether we have read maxPointNumber and calculated maxPointValue. */ private boolean isMaxPointNumberRead; + private List<Boolean> useMaxPointNumber; + private int position = 0; + public FloatDecoder(TSEncoding encodingType, TSDataType dataType) { super(encodingType); if (encodingType == TSEncoding.RLE) { @@ -93,7 +98,8 @@ public class FloatDecoder extends Decoder { public float readFloat(ByteBuffer buffer) { readMaxPointValue(buffer); int value = decoder.readInt(buffer); - double result = value / maxPointValue; + double result = value / getMaxPointValue(); + position++; return (float) result; } @@ -104,10 +110,31 @@ public class FloatDecoder extends Decoder { return value / maxPointValue; } + private double getMaxPointValue() { + if (useMaxPointNumber == null) { + return maxPointValue; + } else { + return useMaxPointNumber.get(position) ? maxPointValue : 1; + } + } + private void readMaxPointValue(ByteBuffer buffer) { if (!isMaxPointNumberRead) { int maxPointNumber = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer); - if (maxPointNumber <= 0) { + if (maxPointNumber == Integer.MAX_VALUE) { + useMaxPointNumber = new ArrayList<>(); + while (true) { + maxPointNumber = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer); + if (maxPointNumber == 1) { + useMaxPointNumber.add(true); + } else if (maxPointNumber == 0) { + useMaxPointNumber.add(false); + } else { + maxPointValue = Math.pow(10, maxPointNumber); + break; + } + } + } else if (maxPointNumber <= 0) { maxPointValue = 1; } else { maxPointValue = Math.pow(10, maxPointNumber); @@ -153,5 +180,6 @@ public class FloatDecoder extends Decoder { public void reset() { this.decoder.reset(); this.isMaxPointNumberRead = false; + this.position = 0; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatEncoder.java index adf328e1..3f254278 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatEncoder.java @@ -26,6 +26,8 @@ import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; /** * Encoder for float or double value using rle or two-diff according to following grammar. @@ -49,11 +51,14 @@ public class FloatEncoder extends Encoder { /** flag to check whether maxPointNumber is saved in the stream. */ private boolean isMaxPointNumberSaved; + private final List<Boolean> useMaxPointNumber; + public FloatEncoder(TSEncoding encodingType, TSDataType dataType, int maxPointNumber) { super(encodingType); this.maxPointNumber = maxPointNumber; - calculateMaxPonitNum(); + calculateMaxPointNum(); isMaxPointNumberSaved = false; + useMaxPointNumber = new ArrayList<>(); if (encodingType == TSEncoding.RLE) { if (dataType == TSDataType.FLOAT) { encoder = new IntRleEncoder(); @@ -101,7 +106,7 @@ public class FloatEncoder extends Encoder { encoder.encode(valueLong, out); } - private void calculateMaxPonitNum() { + private void calculateMaxPointNum() { if (maxPointNumber <= 0) { maxPointNumber = 0; maxPointValue = 1; @@ -111,7 +116,13 @@ public class FloatEncoder extends Encoder { } private int convertFloatToInt(float value) { - return (int) Math.round(value * maxPointValue); + if (value * maxPointValue > Integer.MAX_VALUE || value * maxPointValue < Integer.MIN_VALUE) { + useMaxPointNumber.add(false); + return Math.round(value); + } else { + useMaxPointNumber.add(true); + return (int) Math.round(value * maxPointValue); + } } private long convertDoubleToLong(double value) { @@ -121,11 +132,30 @@ public class FloatEncoder extends Encoder { @Override public void flush(ByteArrayOutputStream out) throws IOException { encoder.flush(out); + if (pointsNotUseMaxPointNumber()) { + byte[] ba = out.toByteArray(); + out.reset(); + ReadWriteForEncodingUtils.writeUnsignedVarInt(Integer.MAX_VALUE, out); + for (boolean b : useMaxPointNumber) { + ReadWriteForEncodingUtils.writeUnsignedVarInt(b ? 1 : 0, out); + } + out.write(ba); + } reset(); } private void reset() { isMaxPointNumberSaved = false; + useMaxPointNumber.clear(); + } + + private boolean pointsNotUseMaxPointNumber() { + for (boolean info : useMaxPointNumber) { + if (!info) { + return true; + } + } + return false; } private void saveMaxPointNumber(ByteArrayOutputStream out) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/FloatDecoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/FloatDecoderTest.java index bdc1db9f..cf631cad 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/FloatDecoderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/FloatDecoderTest.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -84,6 +85,25 @@ public class FloatDecoderTest { @After public void tearDown() {} + public static void main(String[] args) throws IOException { + float value1 = 0.333F; + System.out.println(value1); + float value = 6.5536403E8F; + System.out.println(value); + Encoder encoder = new FloatEncoder(TSEncoding.TS_2DIFF, TSDataType.FLOAT, 2); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + encoder.encode(value1, baos); + encoder.encode(value, baos); + encoder.flush(baos); + + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Decoder decoder = new FloatDecoder(TSEncoding.TS_2DIFF, TSDataType.FLOAT); + float value_ = decoder.readFloat(buffer); + System.out.println(value_); + value_ = decoder.readFloat(buffer); + System.out.println(value_); + } + @Test public void testRLEFloat() throws Exception { for (int i = 1; i <= 10; i++) {
