This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch fix_rlbe_lossy_2 in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 62e8127de938288b599c208351527e3d68e4f510 Author: HTHou <[email protected]> AuthorDate: Wed May 7 10:55:49 2025 +0800 Fix float RLBE encoding loss of precision --- .../tsfile/encoding/decoder/DoubleRLBEDecoder.java | 171 +------------- .../tsfile/encoding/decoder/FloatRLBEDecoder.java | 170 +------------ .../apache/tsfile/encoding/encoder/DoubleRLBE.java | 262 +-------------------- .../apache/tsfile/encoding/encoder/FloatRLBE.java | 244 +------------------ .../tsfile/encoding/decoder/RLBEDecoderTest.java | 83 ++++++- 5 files changed, 83 insertions(+), 847 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleRLBEDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleRLBEDecoder.java index a31be5d0..f3cf946b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleRLBEDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleRLBEDecoder.java @@ -19,180 +19,17 @@ package org.apache.tsfile.encoding.decoder; -import org.apache.tsfile.file.metadata.enums.TSEncoding; - import java.nio.ByteBuffer; -public class DoubleRLBEDecoder extends Decoder { +public class DoubleRLBEDecoder extends LongRLBEDecoder { + /** constructor of DoubleRLBEDecoder */ public DoubleRLBEDecoder() { - super(TSEncoding.RLBE); - numberLeftInBuffer = 0; - byteBuffer = 0; - } - - /** how many numbers expected to decode in following block */ - private int blocksize; - - /** origin values are stored in data */ - private double[] data; - - /** whether the first value is decoded */ - private int writeindex = -1; - - /** the pointer of current last value */ - private int readindex = -1; - - /** fibonacci values are stored in fibonacci */ - private long[] fibonacci; - - /** read bits from input stream to byteBuffer and get bit from byteBuffer */ - private byte byteBuffer; - - /** valid bits in byteBuffer */ - private int numberLeftInBuffer; - - /** - * read the header of a block, determine the size of block and malloc space values - * - * @param buffer inputstream buffer - */ - private void readhead(ByteBuffer buffer) { - for (int i = 0; i <= writeindex; i++) { - data[i] = 0; - } - writeindex = -1; - readindex = -1; - clearBuffer(buffer); - readblocksize(buffer); - data = new double[blocksize * 2 + 1]; - fibonacci = new long[blocksize * 2 + 1]; - for (int i = 0; i < blocksize * 2; i++) { - data[i] = 0; - fibonacci[i] = 0; - } - fibonacci[0] = 1; - fibonacci[1] = 1; - } - - /** - * read a block from inputstream buffer - * - * @param buffer inputstream buffer - */ - private void readT(ByteBuffer buffer) { - // read the header of the block - readhead(buffer); - while (writeindex < blocksize - 1) { - int seglength = 0; - long runlength = 0; - // read first 7 bits: length of each binary words. - for (int j = 6; j >= 0; j--) { - seglength |= (readbit(buffer) << j); - } - - // generate repeat time of rle on delta - int now = readbit(buffer); - int next = readbit(buffer); - - int j = 1; - while (true) { - if (j > 1) fibonacci[j] = fibonacci[j - 1] + fibonacci[j - 2]; - if (now == 1) runlength += fibonacci[j]; - // when now and next are both 1, the 1 of next is the symbol of ending of fibonacci code - if (now == 1 && next == 1) break; - j++; - now = next; - next = readbit(buffer); - } - // read the delta value one by one - for (long i = 1; i <= runlength; i++) { - - long readlongtemp = 0; - for (int k = seglength - 1; k >= 0; k--) { - readlongtemp += ((long) readbit(buffer) << k); - } - if (seglength == 64) readlongtemp -= ((long) 1 << 63); - double readdoubletemp = Double.longBitsToDouble(readlongtemp); - if (writeindex == -1) { - data[++writeindex] = readdoubletemp; - } else { - ++writeindex; - data[writeindex] = data[writeindex - 1] + readdoubletemp; - } - } - } + super(); } @Override public double readDouble(ByteBuffer buffer) { - if (readindex < writeindex) { - return data[++readindex]; - } else { - readT(buffer); - return data[++readindex]; - } - } - - @Override - public boolean hasNext(ByteBuffer buffer) { - return (buffer.remaining() > 0 || readindex < writeindex); - } - - @Override - public void reset() { - // do nothing - } - - /** - * get a bit from byteBuffer, when there is no bit in byteBuffer, get new 8 bits from inputstream - * buffer - * - * @param buffer inputstream buffer - * @return the top bit of byteBuffer - */ - private int readbit(ByteBuffer buffer) { - if (numberLeftInBuffer == 0) { - loadBuffer(buffer); - numberLeftInBuffer = 8; - } - int top = ((byteBuffer >> 7) & 1); - byteBuffer <<= 1; - numberLeftInBuffer--; - return top; - } - - /** - * get 8 bits from inputstream buffer to byteBuffer - * - * @param buffer inputstream buffer - */ - private void loadBuffer(ByteBuffer buffer) { - byteBuffer = buffer.get(); - } - - /** - * clear all remaining bits in byteBuffer - * - * @param buffer inputstream buffer - */ - private void clearBuffer(ByteBuffer buffer) { - while (numberLeftInBuffer > 0) { - readbit(buffer); - } - } - - /** - * read the first integer of the block: blocksize - * - * @param buffer inputstream buffer - */ - private void readblocksize(ByteBuffer buffer) { - blocksize = 0; - for (int i = 31; i >= 0; i--) { - if (readbit(buffer) == 1) { - blocksize |= (1 << i); - } - } + return Double.longBitsToDouble(super.readLong(buffer)); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatRLBEDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatRLBEDecoder.java index eb309e1d..c54ccfda 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatRLBEDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatRLBEDecoder.java @@ -19,179 +19,17 @@ package org.apache.tsfile.encoding.decoder; -import org.apache.tsfile.file.metadata.enums.TSEncoding; - import java.nio.ByteBuffer; -public class FloatRLBEDecoder extends Decoder { +public class FloatRLBEDecoder extends IntRLBEDecoder { + /** constructor of FloatRLBEDecoder */ public FloatRLBEDecoder() { - super(TSEncoding.RLBE); - numberLeftInBuffer = 0; - byteBuffer = 0; - } - - /** how many numbers expected to decode in following block */ - private int blocksize; - - /** origin values are stored in data */ - private float[] data; - - /** whether the first value is decoded */ - private int writeindex = -1; - - /** the pointer of current last value */ - private int readindex = -1; - - /** fibonacci values are stored in fibonacci */ - private int[] fibonacci; - - /** read bits from input stream to byteBuffer and get bit from byteBuffer */ - private byte byteBuffer; - - /** valid bits in byteBuffer */ - private int numberLeftInBuffer; - - /** - * read the header of a block, determine the size of block and malloc space values - * - * @param buffer inputstream buffer - */ - private void readhead(ByteBuffer buffer) { - for (int i = 0; i <= writeindex; i++) { - data[i] = 0; - } - writeindex = -1; - readindex = -1; - clearBuffer(buffer); - readblocksize(buffer); - data = new float[blocksize * 2 + 1]; - fibonacci = new int[blocksize * 2 + 1]; - for (int i = 0; i < blocksize * 2; i++) { - data[i] = 0; - fibonacci[i] = 0; - } - fibonacci[0] = 1; - fibonacci[1] = 1; - } - - /** - * read a block from inputstream buffer - * - * @param buffer inputstream buffer - */ - private void readT(ByteBuffer buffer) { - // read the header of the block - readhead(buffer); - while (writeindex < blocksize - 1) { - int seglength = 0, runlength = 0; - // read first 6 bits: length of each binary words. - for (int j = 5; j >= 0; j--) { - seglength |= (readbit(buffer) << j); - } - - // generate repeat time of rle on delta - int now = readbit(buffer); - int next = readbit(buffer); - - int j = 1; - while (true) { - if (j > 1) fibonacci[j] = fibonacci[j - 1] + fibonacci[j - 2]; - if (now == 1) runlength += fibonacci[j]; - // when now and next are both 1, the 1 of next is the symbol of ending of fibonacci code - if (now == 1 && next == 1) break; - j++; - now = next; - next = readbit(buffer); - } - // read the delta value one by one - for (int i = 1; i <= runlength; i++) { - - int readinttemp = 0; - for (int k = seglength - 1; k >= 0; k--) { - readinttemp += (readbit(buffer) << k); - } - if (seglength == 32) readinttemp -= (1 << 31); - float readfloattemp = Float.intBitsToFloat(readinttemp); - if (writeindex == -1) { - data[++writeindex] = readfloattemp; - } else { - ++writeindex; - data[writeindex] = data[writeindex - 1] + readfloattemp; - } - } - } + super(); } @Override public float readFloat(ByteBuffer buffer) { - if (readindex < writeindex) { - return data[++readindex]; - } else { - readT(buffer); - return data[++readindex]; - } - } - - @Override - public boolean hasNext(ByteBuffer buffer) { - return (buffer.remaining() > 0 || readindex < writeindex); - } - - @Override - public void reset() { - // do nothing - } - - /** - * get a bit from byteBuffer, when there is no bit in byteBuffer, get new 8 bits from inputstream - * buffer - * - * @param buffer inputstream buffer - * @return the top bit of byteBuffer - */ - private int readbit(ByteBuffer buffer) { - if (numberLeftInBuffer == 0) { - loadBuffer(buffer); - numberLeftInBuffer = 8; - } - int top = ((byteBuffer >> 7) & 1); - byteBuffer <<= 1; - numberLeftInBuffer--; - return top; - } - - /** - * get 8 bits from inputstream buffer to byteBuffer - * - * @param buffer inputstream buffer - */ - private void loadBuffer(ByteBuffer buffer) { - byteBuffer = buffer.get(); - } - - /** - * clear all remaining bits in byteBuffer - * - * @param buffer inputstream buffer - */ - private void clearBuffer(ByteBuffer buffer) { - while (numberLeftInBuffer > 0) { - readbit(buffer); - } - } - - /** - * read the first integer of the block: blocksize - * - * @param buffer inputstream buffer - */ - private void readblocksize(ByteBuffer buffer) { - blocksize = 0; - for (int i = 31; i >= 0; i--) { - if (readbit(buffer) == 1) { - blocksize |= (1 << i); - } - } + return Float.intBitsToFloat(super.readInt(buffer)); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleRLBE.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleRLBE.java index 40fc8d86..754d42b0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleRLBE.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleRLBE.java @@ -21,273 +21,15 @@ package org.apache.tsfile.encoding.encoder; import java.io.ByteArrayOutputStream; -public class DoubleRLBE extends RLBE { - // delta values - private final double[] diffValue = new double[blockSize + 1]; - - // repeat times on length code - private final long[] lengRLE = new long[blockSize + 1]; - - // previous value of original value - private double previousValue; +public class DoubleRLBE extends LongRLBE { // constructor of DoubleRLBE public DoubleRLBE() { super(); - reset(); - } - - protected void reset() { - writeIndex = -1; - LengthCode = new int[blockSize + 1]; - for (int i = 0; i < blockSize; i++) { - diffValue[i] = 0; - LengthCode[i] = 0; - byteBuffer = 0; - numberLeftInBuffer = 0; - lengRLE[i] = 0; - } - } - - /** - * calculate the binary code length of given long integer. - * - * @param val the long integer to calculate length - * @return the length of val's binary code - */ - private int calBinarylength(long val) { - if (val == 0) { - return 1; - } - int i = 64; - while ((((long) 1 << (i - 1)) & val) == 0 && i > 0) { - i--; - } - return i; - } - - /** - * calculate the binary code length of given double note: double is transfered into long first. - * - * @param v the double to calculate length - * @return the length of val's binary code - */ - private int calBinarylength(double v) { - long val = Double.doubleToRawLongBits(v); - if (val == 0) { - return 1; - } - int i = 64; - while ((((long) 1 << (i - 1)) & val) == 0 && i > 0) { - i--; - } - return i; - } - - /** - * encode one input integer value. - * - * @param value the integer to be encoded - * @param out the output stream to flush in when buffer is full - */ - public void encodeValue(double value, ByteArrayOutputStream out) { - if (writeIndex == -1) { - // when the first value hasn't encoded yet - diffValue[++writeIndex] = value; - LengthCode[writeIndex] = calBinarylength(value); - previousValue = value; - return; - } - // calculate delta value - diffValue[++writeIndex] = value - previousValue; - // caldulate the length of delta value - LengthCode[writeIndex] = calBinarylength(diffValue[writeIndex]); - previousValue = value; - if (writeIndex == blockSize - 1) { - // when encoded number reach to blocksize - flush(out); - } } @Override public void encode(double value, ByteArrayOutputStream out) { - encodeValue(value, out); - } - - @Override - public void flush(ByteArrayOutputStream out) { - flushBlock(out); - } - - /** - * calculate fibonacci code of input long integer. - * - * @param val the long integer to be fibonacci-encoded - * @return the reverse fibonacci code of val in binary code - */ - protected long calcFibonacci(long val) { - // fibonacci values are stored in Fib - long[] fib = new long[blockSize * 2 + 1]; - fib[0] = 1; - fib[1] = 1; - int i; - // generate fibonacci values from 1 to the first one larger than val - for (i = 2; fib[i - 1] <= val; i++) { - fib[i] = fib[i - 1] + fib[i - 2]; - } - - i--; - long valfib = 0; - // calculate fibonacci code - while (val > 0) { - while (fib[i] > val && i >= 1) { - i--; - } - valfib |= (1 << (i - 1)); - val -= fib[i]; - } - return valfib; - } - - /** run length on DiffValue then store length at the first index in Lengrle. */ - private void rleonlengthcode() { - int i = 0; - while (i <= writeIndex) { - int j = i; - int temprlecal = 0; - while (LengthCode[j] == LengthCode[i] && j <= writeIndex) { - j++; - temprlecal++; - } - // store repeat time at the first repeating value's position - lengRLE[i] = temprlecal; - i = j; - } - } - - /** - * flush all encoded values in a block to output stream. - * - * @param out the output stream to be flushed to - */ - protected void flushBlock(ByteArrayOutputStream out) { - if (writeIndex == -1) { - return; - } - // store the number of values - writewriteIndex(out); - // calculate length code of delta binary length - rleonlengthcode(); - for (int i = 0; i <= writeIndex; i++) { - if (lengRLE[i] > 0) { // flush the adjacent same length delta values - flushSegment(i, out); - } - } - clearBuffer(out); - reset(); - } - - /** - * flush the adjacent same-length delta values. - * - * @param i the position of the first delta value - * @param out output stream - */ - private void flushSegment(int i, ByteArrayOutputStream out) { - // write the first 6 bits: length code in binary words. - for (int j = 6; j >= 0; j--) { - if ((LengthCode[i] & (1 << j)) > 0) { - writeBit(true, out); - } else { - writeBit(false, out); - } - } - // write the fibonacci code in normal direction - long fib = calcFibonacci(lengRLE[i]); - int fiblen = calBinarylength(fib); - for (int j = 0; j < fiblen; j++) { - if ((fib & (1 << j)) > 0) { - writeBit(true, out); - } else { - writeBit(false, out); - } - } - // write '1' to note the end of fibonacci code - writeBit(true, out); - - // write Binary code words - int j = i; - do { - int tempDifflen = calBinarylength(diffValue[j]); - long tempDiff = Double.doubleToRawLongBits(diffValue[j]); - for (int k = tempDifflen - 1; k >= 0; k--) { - if ((tempDiff & ((long) 1 << k)) > 0) { - writeBit(true, out); - } else { - writeBit(false, out); - } - } - j++; - } while (lengRLE[j] == 0 && j <= writeIndex); - } - - @Override - public int getOneItemMaxSize() { - return 4 * 4 * 2; - } - - @Override - public long getMaxByteSize() { - return 5L * 4 * blockSize * 2; - } - - /** - * write one bit to byteBuffer, when byteBuffer is full, flush byteBuffer to output stream - * - * @param b the bit to be written - * @param out output stream - */ - protected void writeBit(boolean b, ByteArrayOutputStream out) { - byteBuffer <<= 1; - if (b) { - byteBuffer |= 1; - } - - numberLeftInBuffer++; - if (numberLeftInBuffer == 8) { - clearBuffer(out); - } - } - - /** - * flush bits left in byteBuffer to output stream. - * - * @param out output stream - */ - protected void clearBuffer(ByteArrayOutputStream out) { - if (numberLeftInBuffer == 0) { - return; - } - if (numberLeftInBuffer > 0) { - byteBuffer <<= (8 - numberLeftInBuffer); - } - out.write(byteBuffer); - numberLeftInBuffer = 0; - byteBuffer = 0; - } - - /** - * write the number of encoded values to output stream - * - * @param out output stream - */ - private void writewriteIndex(ByteArrayOutputStream out) { - for (int i = 31; i >= 0; i--) { - if ((writeIndex + 1 & (1 << i)) > 0) { - writeBit(true, out); - } else { - writeBit(false, out); - } - } + super.encode(Double.doubleToRawLongBits(value), out); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatRLBE.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatRLBE.java index 43f4ac17..3d6b763a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatRLBE.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatRLBE.java @@ -20,254 +20,16 @@ package org.apache.tsfile.encoding.encoder; import java.io.ByteArrayOutputStream; -import java.io.IOException; -public class FloatRLBE extends RLBE { - /** delta values */ - private float[] DiffValue = new float[blockSize + 1]; +public class FloatRLBE extends IntRLBE { - /** repeat times on length code */ - private int[] Lengrle = new int[blockSize + 1]; - - /** previous value of original value */ - private float previousvalue; - - /** constructor of FloatRLBE */ + // constructor of FloatRLBE public FloatRLBE() { super(); - reset(); - } - - protected void reset() { - writeIndex = -1; - LengthCode = new int[blockSize + 1]; - for (int i = 0; i < blockSize; i++) { - DiffValue[i] = 0; - LengthCode[i] = 0; - byteBuffer = 0; - numberLeftInBuffer = 0; - Lengrle[i] = 0; - } - } - - /** - * calculate the binary code length of given float note: binary code of float is transfered into - * binary code of int first - * - * @param v the integer to calculate length - * @return the length of val's binary code - */ - private int calBinarylength(float v) { - int val = Float.floatToRawIntBits(v); - if (val == 0) return 1; - int i = 32; - while (((1 << (i - 1)) & val) == 0 && i > 0) i--; - return i; - } - - /** - * calculate the binary code length of given integer - * - * @param val the integer to calculate length - * @return the length of val's binary code - */ - private int calBinarylength(int val) { - if (val == 0) return 1; - int i = 32; - while (((1 << (i - 1)) & val) == 0 && i > 0) i--; - return i; - } - - /** - * encode one input float value - * - * @param value the float to be encoded - * @param out the output stream to flush in when buffer is full - */ - public void encodeValue(float value, ByteArrayOutputStream out) { - if (writeIndex == -1) { - // when the first value hasn't encoded yet - DiffValue[++writeIndex] = value; - LengthCode[writeIndex] = calBinarylength(value); - previousvalue = value; - return; - } - // calculate delta value - DiffValue[++writeIndex] = value - previousvalue; - // caldulate the length of delta value - LengthCode[writeIndex] = calBinarylength(DiffValue[writeIndex]); - previousvalue = value; - if (writeIndex == blockSize - 1) { - // when encoded number reach to blocksize - flush(out); - } } @Override public void encode(float value, ByteArrayOutputStream out) { - encodeValue(value, out); - } - - @Override - public void flush(ByteArrayOutputStream out) { - flushBlock(out); - } - - /** - * calculate fibonacci code of input integer - * - * @param val the integer to be fibonacci-encoded - * @return the reverse fibonacci code of val in binary code - */ - protected int calcFibonacci(int val) { - // fibonacci values are stored in Fib - int[] Fib = new int[blockSize * 2 + 1]; - Fib[0] = 1; - Fib[1] = 1; - int i; - // generate fibonacci values from 1 to the first one larger than val - for (i = 2; Fib[i - 1] <= val; i++) { - Fib[i] = Fib[i - 1] + Fib[i - 2]; - } - - i--; - int valfib = 0; - // calculate fibonacci code - while (val > 0) { - while (Fib[i] > val && i >= 1) i--; - valfib |= (1 << (i - 1)); - val -= Fib[i]; - } - return valfib; - } - - /** run length on DiffValue then store length at the first index in Lengrle. */ - private void rleonlengthcode() { - int i = 0; - while (i <= writeIndex) { - int j = i; - int temprlecal = 0; - while (LengthCode[j] == LengthCode[i] && j <= writeIndex) { - j++; - temprlecal++; - } - // store repeat time at the first repeating value's position - Lengrle[i] = temprlecal; - i = j; - } - } - - /** - * flush all encoded values in a block to output stream - * - * @param out the output stream to be flushed to - */ - protected void flushBlock(ByteArrayOutputStream out) { - if (writeIndex == -1) { - return; - } - // store the number of values - writewriteIndex(out); - // calculate length code of delta binary length - rleonlengthcode(); - for (int i = 0; i <= writeIndex; i++) { - if (Lengrle[i] > 0) // flush the adjacent same length delta values - try { - flushsegment(i, out); - } catch (IOException e) { - logger.error("flush data to stream failed!", e); - } - } - clearBuffer(out); - reset(); - } - - /** - * flush the adjacent same-length delta values - * - * @param i the position of the first delta value - * @param out output stream - * @throws IOException - */ - private void flushsegment(int i, ByteArrayOutputStream out) throws IOException { - // write the first 6 bits: length code in binary words. - for (int j = 5; j >= 0; j--) { - if ((LengthCode[i] & (1 << j)) > 0) writeBit(true, out); - else writeBit(false, out); - } - // write the fibonacci code in normal direction - int fib = calcFibonacci(Lengrle[i]); - int fiblen = calBinarylength(fib); - for (int j = 0; j < fiblen; j++) { - if ((fib & (1 << j)) > 0) writeBit(true, out); - else writeBit(false, out); - } - // write '1' to note the end of fibonacci code - writeBit(true, out); - - // write Binary code words - int j = i; - do { - int tempDifflen = calBinarylength(DiffValue[j]); - int tempDiff = Float.floatToRawIntBits(DiffValue[j]); - for (int k = tempDifflen - 1; k >= 0; k--) { - if ((tempDiff & (1 << k)) > 0) writeBit(true, out); - else writeBit(false, out); - } - j++; - } while (Lengrle[j] == 0 && j <= writeIndex); - } - - @Override - public int getOneItemMaxSize() { - return 4 * 4; - } - - @Override - public long getMaxByteSize() { - return 5 * 4 * blockSize; - } - - /** - * write one bit to byteBuffer, when byteBuffer is full, flush byteBuffer to output stream - * - * @param b the bit to be written - * @param out output stream - */ - protected void writeBit(boolean b, ByteArrayOutputStream out) { - byteBuffer <<= 1; - if (b) { - byteBuffer |= 1; - } - - numberLeftInBuffer++; - if (numberLeftInBuffer == 8) { - clearBuffer(out); - } - } - - /** - * flush bits left in byteBuffer to output stream - * - * @param out output stream - */ - protected void clearBuffer(ByteArrayOutputStream out) { - if (numberLeftInBuffer == 0) return; - if (numberLeftInBuffer > 0) byteBuffer <<= (8 - numberLeftInBuffer); - out.write(byteBuffer); - numberLeftInBuffer = 0; - byteBuffer = 0; - } - - /** - * write the number of encoded values to output stream - * - * @param out output stream - */ - private void writewriteIndex(ByteArrayOutputStream out) { - for (int i = 31; i >= 0; i--) { - if ((writeIndex + 1 & (1 << i)) > 0) writeBit(true, out); - else writeBit(false, out); - } + super.encode(Float.floatToRawIntBits(value), out); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java index c212f2a2..bdb8082e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java @@ -43,7 +43,8 @@ import static org.junit.Assert.fail; public class RLBEDecoderTest { private static final Logger logger = LoggerFactory.getLogger(RLBEDecoderTest.class); - private final double delta = 0.0000001; + private final float floatDelta = 0; + private final double doubleDelta = 0; private final int floatMaxPointValue = 10000; private final long doubleMaxPointValue = 1000000000000000L; private List<Float> floatList; @@ -105,13 +106,13 @@ public class RLBEDecoderTest { for (int i = 0; i < 2; i++) { Decoder decoder = new DoubleRLBEDecoder(); if (decoder.hasNext(buffer)) { - assertEquals(value, decoder.readDouble(buffer), delta); + assertEquals(value, decoder.readDouble(buffer), doubleDelta); } if (decoder.hasNext(buffer)) { - assertEquals(value - 2, decoder.readDouble(buffer), delta); + assertEquals(value - 2, decoder.readDouble(buffer), doubleDelta); } if (decoder.hasNext(buffer)) { - assertEquals(value - 4, decoder.readDouble(buffer), delta); + assertEquals(value - 4, decoder.readDouble(buffer), doubleDelta); } } } @@ -133,13 +134,13 @@ public class RLBEDecoderTest { for (int i = 0; i < 2; i++) { Decoder decoder = new DoubleRLBEDecoder(); if (decoder.hasNext(buffer)) { - assertEquals(value, decoder.readDouble(buffer), delta); + assertEquals(value, decoder.readDouble(buffer), doubleDelta); } if (decoder.hasNext(buffer)) { - assertEquals(value, decoder.readDouble(buffer), delta); + assertEquals(value, decoder.readDouble(buffer), doubleDelta); } if (decoder.hasNext(buffer)) { - assertEquals(value, decoder.readDouble(buffer), delta); + assertEquals(value, decoder.readDouble(buffer), doubleDelta); } } } @@ -172,17 +173,45 @@ public class RLBEDecoderTest { Decoder decoder = new FloatRLBEDecoder(); for (int i = 0; i < num; i++) { if (decoder.hasNext(buffer)) { - assertEquals(value + 2 * i, decoder.readFloat(buffer), delta); + assertEquals(value + 2 * i, decoder.readFloat(buffer), floatDelta); continue; } fail(); } } + @Test + public void testFloat2() throws Exception { + float a = 934.02F; + float b = 122.86F; + float c = 33.15F; + float d = 33.15F; + float f = Float.NaN; + float e = Float.POSITIVE_INFINITY; + Encoder encoder = new FloatRLBE(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + encoder.encode(a, baos); + encoder.encode(b, baos); + encoder.encode(c, baos); + encoder.encode(d, baos); + encoder.encode(e, baos); + encoder.encode(f, baos); + encoder.flush(baos); + + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Decoder decoder = new FloatRLBEDecoder(); + assertEquals(a, decoder.readFloat(buffer), floatDelta); + assertEquals(b, decoder.readFloat(buffer), floatDelta); + assertEquals(c, decoder.readFloat(buffer), floatDelta); + assertEquals(d, decoder.readFloat(buffer), floatDelta); + assertEquals(e, decoder.readFloat(buffer), floatDelta); + assertEquals(f, decoder.readFloat(buffer), floatDelta); + } + @Test public void testDouble() throws IOException { Encoder encoder = - TSEncodingBuilder.RLBE.getEncodingBuilder(TSEncoding.RLBE).getEncoder(TSDataType.DOUBLE); + TSEncodingBuilder.getEncodingBuilder(TSEncoding.RLBE).getEncoder(TSDataType.DOUBLE); ByteArrayOutputStream baos = new ByteArrayOutputStream(); double value = 7.101f; int num = 1000; @@ -194,13 +223,41 @@ public class RLBEDecoderTest { Decoder decoder = Decoder.getDecoderByType(TSEncoding.RLBE, TSDataType.DOUBLE); for (int i = 0; i < num; i++) { if (decoder.hasNext(buffer)) { - assertEquals(value + 2 * i, decoder.readDouble(buffer), delta); + assertEquals(value + 2 * i, decoder.readDouble(buffer), doubleDelta); continue; } fail(); } } + @Test + public void testDouble2() throws Exception { + double a = 934.02; + double b = 122.86; + double c = 33.15; + double d = 33.15; + double f = Double.NaN; + double e = Double.POSITIVE_INFINITY; + Encoder encoder = new DoubleRLBE(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + encoder.encode(a, baos); + encoder.encode(b, baos); + encoder.encode(c, baos); + encoder.encode(d, baos); + encoder.encode(e, baos); + encoder.encode(f, baos); + encoder.flush(baos); + + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + Decoder decoder = new DoubleRLBEDecoder(); + assertEquals(a, decoder.readDouble(buffer), doubleDelta); + assertEquals(b, decoder.readDouble(buffer), doubleDelta); + assertEquals(c, decoder.readDouble(buffer), doubleDelta); + assertEquals(d, decoder.readDouble(buffer), doubleDelta); + assertEquals(e, decoder.readDouble(buffer), doubleDelta); + assertEquals(f, decoder.readDouble(buffer), doubleDelta); + } + private void testFloatLength(List<Float> valueList, boolean isDebug, int repeatCount) throws Exception { Encoder encoder = new FloatRLBE(); @@ -221,7 +278,7 @@ public class RLBEDecoderTest { if (isDebug) { logger.debug("{} // {}", value_, value); } - assertEquals(value, value_, delta); + assertEquals(value, value_, floatDelta); continue; } fail(); @@ -232,7 +289,7 @@ public class RLBEDecoderTest { private void testDoubleLength(List<Double> valueList, boolean isDebug, int repeatCount) throws Exception { Encoder encoder = - TSEncodingBuilder.RLBE.getEncodingBuilder(TSEncoding.RLBE).getEncoder(TSDataType.DOUBLE); + TSEncodingBuilder.getEncodingBuilder(TSEncoding.RLBE).getEncoder(TSDataType.DOUBLE); ByteArrayOutputStream baos = new ByteArrayOutputStream(); for (int i = 0; i < repeatCount; i++) { for (double value : valueList) { @@ -251,7 +308,7 @@ public class RLBEDecoderTest { if (isDebug) { logger.debug("{} // {}", value_, value); } - assertEquals(value, value_, delta); + assertEquals(value, value_, doubleDelta); continue; } fail();
