This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch fix_rlbe_lossy_ in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 79c355e1148020465a4d4a7bafef227a8c379bf2 Author: HTHou <[email protected]> AuthorDate: Tue Apr 29 10:26:21 2025 +0800 dev --- .../tsfile/encoding/decoder/DoubleRLBEDecoder.java | 4 +- .../apache/tsfile/encoding/encoder/DoubleRLBE.java | 4 +- .../tsfile/utils/ReadWriteForEncodingUtils.java | 59 ++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleRLBEDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleRLBEDecoder.java index a31be5d0..fbe3e4e5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleRLBEDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleRLBEDecoder.java @@ -23,6 +23,8 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import java.nio.ByteBuffer; +import static org.apache.tsfile.utils.ReadWriteForEncodingUtils.addAccurate; + public class DoubleRLBEDecoder extends Decoder { /** constructor of DoubleRLBEDecoder */ public DoubleRLBEDecoder() { @@ -118,7 +120,7 @@ public class DoubleRLBEDecoder extends Decoder { data[++writeindex] = readdoubletemp; } else { ++writeindex; - data[writeindex] = data[writeindex - 1] + readdoubletemp; + data[writeindex] = addAccurate(data[writeindex - 1], readdoubletemp); } } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleRLBE.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleRLBE.java index 40fc8d86..0c8861b0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleRLBE.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleRLBE.java @@ -21,6 +21,8 @@ package org.apache.tsfile.encoding.encoder; import java.io.ByteArrayOutputStream; +import static org.apache.tsfile.utils.ReadWriteForEncodingUtils.subAccurate; + public class DoubleRLBE extends RLBE { // delta values private final double[] diffValue = new double[blockSize + 1]; @@ -99,7 +101,7 @@ public class DoubleRLBE extends RLBE { return; } // calculate delta value - diffValue[++writeIndex] = value - previousValue; + diffValue[++writeIndex] = subAccurate(value, previousValue); // caldulate the length of delta value LengthCode[writeIndex] = calBinarylength(diffValue[writeIndex]); previousValue = value; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteForEncodingUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteForEncodingUtils.java index 15940db5..4f19f841 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteForEncodingUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteForEncodingUtils.java @@ -24,6 +24,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.nio.ByteBuffer; import java.util.List; @@ -350,4 +352,61 @@ public class ReadWriteForEncodingUtils { } return result; } + + public static int countDecimalPlaces(double num) { + String text = Double.toString(num); + int index = text.indexOf('.'); + if (index < 0) { + return 0; + } + return text.length() - index - 1; + } + + public static int countDecimalPlaces(float num) { + String text = Double.toString(num); + int index = text.indexOf('.'); + if (index < 0) { + return 0; + } + return text.length() - index - 1; + } + + public static double addAccurate(double a, double b) { + if (a == 0 || b == 0 || Double.isNaN(a) || Double.isNaN(b)) { + return a + b; + } + int scale = Math.max(countDecimalPlaces(a), countDecimalPlaces(b)); + BigDecimal result = BigDecimal.valueOf(a).add(BigDecimal.valueOf(b)); + return result.setScale(scale, RoundingMode.HALF_UP).doubleValue(); + } + + public static double subAccurate(double a, double b) { + if (a == b) { + return 0; + } + if (Double.isNaN(a) || Double.isNaN(b)) { + return Double.NaN; + } + int scale = Math.max(countDecimalPlaces(a), countDecimalPlaces(b)); + BigDecimal result = BigDecimal.valueOf(a).subtract(BigDecimal.valueOf(b)); + return result.setScale(scale, RoundingMode.HALF_UP).doubleValue(); + } + + public static float addAccurate(float a, float b) { + if (a == 0 || b == 0) { + return a + b; + } + int scale = Math.max(countDecimalPlaces(a), countDecimalPlaces(b)); + BigDecimal result = BigDecimal.valueOf(a).add(BigDecimal.valueOf(b)); + return result.setScale(scale, RoundingMode.HALF_UP).floatValue(); + } + + public static float subAccurate(float a, float b) { + if (a == b) { + return 0; + } + int scale = Math.max(countDecimalPlaces(a), countDecimalPlaces(b)); + BigDecimal result = BigDecimal.valueOf(a).subtract(BigDecimal.valueOf(b)); + return result.setScale(scale, RoundingMode.HALF_UP).floatValue(); + } }
