This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch research/encoding-reorder in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 253d02f1e23edbba3161016769814bd2f1f83242 Author: xjz17 <[email protected]> AuthorDate: Mon Dec 4 21:37:41 2023 +0800 Update RegerPFloatTest.java --- .../iotdb/tsfile/encoding/RegerPFloatTest.java | 957 ++------------------- 1 file changed, 58 insertions(+), 899 deletions(-) diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java index 14edb0a50b2..65944c63eee 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java @@ -2,8 +2,6 @@ package org.apache.iotdb.tsfile.encoding; import com.csvreader.CsvReader; import com.csvreader.CsvWriter; -import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; -import org.apache.commons.math3.stat.regression.OLSMultipleLinearRegression; import org.apache.commons.math3.linear.*; import org.junit.Test; @@ -79,75 +77,6 @@ public class RegerPFloatTest { return newArray; } - public static int bytes2Integer(byte[] encoded, int start, int num) { - int value = 0; - if (num > 4) { - System.out.println("bytes2Integer error"); - return 0; - } - for (int i = 0; i < num; i++) { - value <<= 8; - int b = encoded[i + start] & 0xFF; - value |= b; - } - return value; - } - - private static int byte2Integer(byte[] encoded, int decode_pos) { - int value = 0; - int b = encoded[decode_pos] & 0xFF; - value |= b; - if (value == 0) return 256; - return value % 256; - } - - public static void pack8Values( - ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) { - int bufIdx = 0; - int valueIdx = offset; - // remaining bits for the current unfinished Integer - int leftBit = 0; - - while (valueIdx < 8 + offset) { - // buffer is used for saving 32 bits as a part of result - int buffer = 0; - // remaining size of bits in the 'buffer' - int leftSize = 32; - - // encode the left bits of current Integer to 'buffer' - if (leftBit > 0) { - buffer |= (values.get(valueIdx) << (32 - leftBit)); - leftSize -= leftBit; - leftBit = 0; - valueIdx++; - } - - while (leftSize >= width && valueIdx < 8 + offset) { - // encode one Integer to the 'buffer' - buffer |= (values.get(valueIdx) << (leftSize - width)); - leftSize -= width; - valueIdx++; - } - // If the remaining space of the buffer can not save the bits for one Integer, - if (leftSize > 0 && valueIdx < 8 + offset) { - // put the first 'leftSize' bits of the Integer into remaining space of the - // buffer - buffer |= (values.get(valueIdx) >>> (width - leftSize)); - leftBit = width - leftSize; - } - - // put the buffer into the final result - for (int j = 0; j < 4; j++) { - encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); - encode_pos++; - bufIdx++; - if (bufIdx >= width) { - return; - } - } - } - // return encode_pos; - } public static void pack8Values( int[][] values, int index, int offset, int width, int encode_pos, byte[] encoded_result) { @@ -197,38 +126,6 @@ public class RegerPFloatTest { // return encode_pos; } - public static int unpack8Values( - byte[] encoded, int offset, int width, int value_pos, int[] result_list) { - int byteIdx = offset; - // int pos_encode = 0; - long buffer = 0; - // total bits which have read from 'buf' to 'buffer'. i.e., - // number of available bits to be decoded. - int totalBits = 0; - int valueIdx = 0; - - while (valueIdx < 8) { - // If current available bits are not enough to decode one Integer, - // then add next byte from buf to 'buffer' until totalBits >= width - while (totalBits < width) { - buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); - byteIdx++; - totalBits += 8; - } - - // If current available bits are enough to decode one Integer, - // then decode one Integer one by one until left bits in 'buffer' is - // not enough to decode one Integer. - while (totalBits >= width && valueIdx < 8) { - result_list[value_pos] = (int) (buffer >>> (totalBits - width)); - value_pos++; - valueIdx++; - totalBits -= width; - buffer = buffer & ((1L << totalBits) - 1); - } - } - return value_pos; - } public static int bitPacking( int[][] numbers, @@ -247,19 +144,6 @@ public class RegerPFloatTest { return encode_pos; } - public static int decodeBitPacking( - byte[] encoded, int decode_pos, int bit_width, int block_size, int[] result_list) { - // int[] result_list = new int[]; - int block_num = block_size / 8; - int value_pos = 0; - - for (int i = 0; i < block_num; i++) { // bitpacking - value_pos = unpack8Values(encoded, decode_pos, bit_width, value_pos, result_list); - decode_pos += bit_width; - } - // decode_pos_result.add(decode_pos); - return decode_pos; - } public static void float2bytes(float f, int pos_encode, byte[] encode_result) { int fbit = Float.floatToIntBits(f); @@ -278,106 +162,6 @@ public class RegerPFloatTest { } } - public static float bytes2float(ArrayList<Byte> b, int index) { - int l; - l = b.get(index); - l &= 0xff; - l |= ((long) b.get(index + 1) << 8); - l &= 0xffff; - l |= ((long) b.get(index + 2) << 16); - l &= 0xffffff; - l |= ((long) b.get(index + 3) << 24); - return Float.intBitsToFloat(l); - } - - public static float bytes2float(byte[] b, int index) { - int l; - l = b[index]; - l &= 0xff; - l |= ((long) b[index + 1] << 8); - l &= 0xffff; - l |= ((long) b[index + 2] << 16); - l &= 0xffffff; - l |= ((long) b[index + 3] << 24); - return Float.intBitsToFloat(l); - } - - public static byte[] int2Bytes(int integer) { - byte[] bytes = new byte[4]; - bytes[0] = (byte) (integer >> 24); - bytes[1] = (byte) (integer >> 16); - bytes[2] = (byte) (integer >> 8); - bytes[3] = (byte) integer; - return bytes; - } - - public static byte[] bitWidth2Bytes(int integer) { - byte[] bytes = new byte[1]; - bytes[0] = (byte) integer; - return bytes; - } - - public static byte[] float2bytes(float f) { - int fbit = Float.floatToIntBits(f); - byte[] b = new byte[4]; - for (int i = 0; i < 4; i++) { - b[i] = (byte) (fbit >> (24 - i * 8)); - } - int len = b.length; - byte[] dest = new byte[len]; - System.arraycopy(b, 0, dest, 0, len); - byte temp; - for (int i = 0; i < len / 2; ++i) { - temp = dest[i]; - dest[i] = dest[len - i - 1]; - dest[len - i - 1] = temp; - } - return dest; - } - - public static byte[] bitPacking( - ArrayList<ArrayList<Integer>> numbers, int index, int start, int block_num, int bit_width) { - block_num = block_num / 8; - byte[] result = new byte[bit_width * block_num]; - - for (int i = 0; i < block_num; i++) { - for (int j = 0; j < bit_width; j++) { - int tmp_int = 0; - for (int k = 0; k < 8; k++) { - tmp_int += (((numbers.get(start + i * 8 + k).get(index) >> j) % 2) << k); - } - result[i * bit_width + j] = (byte) tmp_int; - } - } - return result; - } - - public static ArrayList<Integer> decodebitPacking( - ArrayList<Byte> encoded, int decode_pos, int bit_width, int min_delta, int block_size) { - ArrayList<Integer> result_list = new ArrayList<>(); - for (int i = 0; i < (block_size - 1) / 8; i++) { - int[] val8 = new int[8]; - for (int j = 0; j < 8; j++) { - val8[j] = 0; - } - for (int j = 0; j < bit_width; j++) { - byte tmp_byte = encoded.get(decode_pos + bit_width - 1 - j); - byte[] bit8 = new byte[8]; - for (int k = 0; k < 8; k++) { - bit8[k] = (byte) (tmp_byte & 1); - tmp_byte = (byte) (tmp_byte >> 1); - } - for (int k = 0; k < 8; k++) { - val8[k] = val8[k] * 2 + bit8[k]; - } - } - for (int j = 0; j < 8; j++) { - result_list.add(val8[j] + min_delta); - } - decode_pos += bit_width; - } - return result_list; - } public static int part(ArrayList<ArrayList<Integer>> arr, int index, int low, int high) { ArrayList<Integer> tmp = arr.get(low); @@ -414,7 +198,7 @@ public class RegerPFloatTest { stack.push(low); stack.push(mid - 1); } - while (stack.empty() == false) { + while (!stack.empty()) { high = stack.pop(); low = stack.pop(); mid = part(arr, index, low, high); @@ -448,10 +232,8 @@ public class RegerPFloatTest { if (td_common == 0) { if (time_diffi != 0) { td_common = time_diffi; - continue; - } else { - continue; } + continue; } if (time_diffi != 0) { td_common = getCommon(time_diffi, td_common); @@ -483,124 +265,11 @@ public class RegerPFloatTest { result.add(t0); } - public static void terminate( - ArrayList<ArrayList<Integer>> ts_block, ArrayList<Float> coefficient, int p) { - int length = ts_block.size(); - assert length > p; - int size = length - p; - - double[] param; - try { - OLSMultipleLinearRegression ols1 = new OLSMultipleLinearRegression(); - double[][] X1 = new double[size][p]; - double[] Y1 = new double[size]; - for (int i = 0; i < size; i++) { - X1[i] = new double[p]; - for (int j = 0; j < p; j++) { - X1[i][j] = ts_block.get(i + j).get(0); - } - Y1[i] = ts_block.get(i + p).get(0); - } - ols1.newSampleData(Y1, X1); - param = ols1.estimateRegressionParameters(); // 结果的第1项是常数项， 之后依次序为各个特征的系数 - // System.out.println(Arrays.toString(param)); - } catch (Exception e) { - param = new double[p + 1]; - for (int i = 0; i <= p; i++) { - param[i] = 0; - } - } - - double[] param2; - try { - OLSMultipleLinearRegression ols2 = new OLSMultipleLinearRegression(); - double[][] X2 = new double[size][p]; - double[] Y2 = new double[size]; - for (int i = 0; i < size; i++) { - X2[i] = new double[p]; - for (int j = 0; j < p; j++) { - X2[i][j] = ts_block.get(i + j).get(1); - } - Y2[i] = ts_block.get(i + p).get(1); - } - ols2.newSampleData(Y2, X2); - param2 = ols2.estimateRegressionParameters(); // 结果的第1项是常数项， 之后依次序为各个特征的系数 - // System.out.println(Arrays.toString(param2)); - } catch (Exception exception) { - param2 = new double[p + 1]; - for (int i = 0; i <= p; i++) { - param2[i] = 0; - } - } - - for (int i = 0; i <= p; i++) { - coefficient.add((float) param[i]); - coefficient.add((float) param2[i]); - } - } - -// public static void terminate(int[][] ts_block, float[] coefficient, int p) { -// int length = ts_block.length; -// assert length > p; -// int size = length - p; -// -// double[] param; -// try { -// OLSMultipleLinearRegression ols1 = new OLSMultipleLinearRegression(); -// double[][] X1 = new double[size][p]; -// double[] Y1 = new double[size]; -// for (int i = 0; i < size; i++) { -// X1[i] = new double[p]; -// for (int j = 0; j < p; j++) { -// X1[i][j] = ts_block[i + j][0]; -// } -// Y1[i] = ts_block[i + p][0]; -// } -// ols1.newSampleData(Y1, X1); -// param = ols1.estimateRegressionParameters(); // 结果的第1项是常数项， 之后依次序为各个特征的系数 -// // System.out.println(Arrays.toString(param)); -// } catch (Exception e) { -// param = new double[p + 1]; -// for (int i = 0; i <= p; i++) { -// param[i] = 0; -// } -// } -// -// double[] param2; -// try { -// OLSMultipleLinearRegression ols2 = new OLSMultipleLinearRegression(); -// double[][] X2 = new double[size][p]; -// double[] Y2 = new double[size]; -// for (int i = 0; i < size; i++) { -// X2[i] = new double[p]; -// for (int j = 0; j < p; j++) { -// X2[i][j] = ts_block[i + j][1]; -// } -// Y2[i] = ts_block[i + p][1]; -// } -// ols2.newSampleData(Y2, X2); -// param2 = ols2.estimateRegressionParameters(); // 结果的第1项是常数项， 之后依次序为各个特征的系数 -// // System.out.println(Arrays.toString(param2)); -// } catch (Exception exception) { -// param2 = new double[p + 1]; -// for (int i = 0; i <= p; i++) { -// param2[i] = 0; -// } -// } -// -// for (int i = 0; i <= p; i++) { -// coefficient[2 * i] = (float) param[i]; -// coefficient[2 * i + 1] = (float) param2[i]; -// } -// } - public static void terminate(int[][] ts_block, float[] coefficient, int p){ int length = ts_block.length; assert length > p; int size = length - p; - OLSMultipleLinearRegression regression = new OLSMultipleLinearRegression(); - RealMatrix matrix = new Array2DRowRealMatrix(size, p+1); RealVector vector = new ArrayRealVector(size); @@ -631,12 +300,6 @@ public class RegerPFloatTest { DecompositionSolver solver1 = new SingularValueDecomposition(matrix).getSolver(); RealVector solution2 = solver1.solve(vector); -// // 输出系数 -// System.out.println("自回归模型系数："); -// for (int i = 0; i < solution.getDimension(); i++) { -// System.out.println("AR(" + (i + 1) + "): " + solution.getEntry(i)); -// } - for (int i = 0; i <= p; i++) { coefficient[2 * i] = (float) solution1.getEntry(p-i); @@ -722,19 +385,13 @@ public class RegerPFloatTest { public static int getBetaP( int[][] ts_block, int alpha, int block_size, int[] raw_length, float[] theta, int p) { - int timestamp_delta_min = Integer.MAX_VALUE; - int value_delta_min = Integer.MAX_VALUE; - int max_timestamp = Integer.MIN_VALUE; - int max_value = Integer.MIN_VALUE; - int raw_timestamp_delta_max_index = -1; - int raw_value_delta_max_index = -1; int raw_abs_sum = raw_length[0]; int range = block_size / 32; ArrayList<Integer> j_star_list = new ArrayList<>(); // beta list of min b phi alpha to j -// ArrayList<Integer> max_index = new ArrayList<>(); + int j_star = -1; @@ -1000,7 +657,6 @@ public class RegerPFloatTest { private static int[] adjustCase1(int[][] ts_block, int alpha, int j_star, float[] theta, int p) { int[][] tmp_ts_block = new int[ts_block.length][2]; cloneTsblock(ts_block,tmp_ts_block); - int block_size = ts_block.length; int[] delta_index = new int[4]; int[] delta = new int[4]; @@ -1036,7 +692,6 @@ public class RegerPFloatTest { private static int[] adjustCase2(int[][] ts_block, int alpha, int j_star, float[] theta, int p) { int[][] tmp_ts_block = new int[ts_block.length][2]; cloneTsblock(ts_block,tmp_ts_block); - int block_size = ts_block.length; int[] tmp_tv = tmp_ts_block[alpha].clone(); for (int u = alpha - 1; u >= j_star; u--) { tmp_ts_block[u + 1][0] = tmp_ts_block[u][0]; @@ -1047,7 +702,7 @@ public class RegerPFloatTest { int[] delta_index = new int[4]; int[] delta = new int[4]; - int[][] ts_block_delta = new int[block_size - p][2]; + int[][] ts_block_delta; ts_block_delta =deltaTSBlock(tmp_ts_block, delta_index,delta, theta, p); int timestamp_delta_min = delta[2]; @@ -1067,7 +722,6 @@ public class RegerPFloatTest { private static int[] adjustCase3(int[][] ts_block, int alpha, int j_star, float[] theta, int p) { int[][] tmp_ts_block = new int[ts_block.length][2]; cloneTsblock(ts_block,tmp_ts_block); - int block_size = ts_block.length; int[] tmp_tv = tmp_ts_block[alpha].clone(); for (int u = alpha + 1; u < j_star; u++) { tmp_ts_block[u - 1][0] = tmp_ts_block[u][0]; @@ -1079,7 +733,7 @@ public class RegerPFloatTest { int[] delta_index = new int[4]; int[] delta = new int[4]; - int[][] ts_block_delta = new int[block_size - p][2]; + int[][] ts_block_delta; ts_block_delta =deltaTSBlock(tmp_ts_block, delta_index,delta, theta, p); int timestamp_delta_min = delta[2]; @@ -1099,7 +753,6 @@ public class RegerPFloatTest { private static int[] adjustCase4(int[][] ts_block, int alpha, int j_star, float[] theta, int p) { int[][] tmp_ts_block = new int[ts_block.length][2]; cloneTsblock(ts_block,tmp_ts_block); - int block_size = ts_block.length; int[] tmp_tv = tmp_ts_block[alpha].clone(); for (int u = alpha + 1; u < j_star; u++) { tmp_ts_block[u - 1][0] = tmp_ts_block[u][0]; @@ -1111,7 +764,7 @@ public class RegerPFloatTest { int[] delta_index = new int[4]; int[] delta = new int[4]; - int[][] ts_block_delta = new int[block_size - p][2]; + int[][] ts_block_delta; ts_block_delta =deltaTSBlock(tmp_ts_block, delta_index,delta, theta, p); int timestamp_delta_min = delta[2]; @@ -1141,7 +794,7 @@ public class RegerPFloatTest { tmp_ts_block[block_size - 1][1] = tmp_tv[1]; int[] delta_index = new int[4]; int[] delta = new int[4]; - int[][] ts_block_delta = new int[block_size - p][2]; + int[][] ts_block_delta; ts_block_delta =deltaTSBlock(tmp_ts_block, delta_index,delta, theta, p); int timestamp_delta_min = delta[2]; @@ -1162,16 +815,11 @@ public class RegerPFloatTest { public static int encodeRLEBitWidth2Bytes(int[][] bit_width_segments) { int encoded_result = 0; - // ArrayList<ArrayList<Integer>> run_length_time = new ArrayList<>(); - // ArrayList<ArrayList<Integer>> run_length_value = new ArrayList<>(); - int count_of_time = 1; int count_of_value = 1; int pre_time = bit_width_segments[0][0]; int pre_value = bit_width_segments[0][1]; int size = bit_width_segments.length; - int[][] run_length_time = new int[size][2]; - int[][] run_length_value = new int[size][2]; int pos_time = 0; int pos_value = 0; @@ -1180,8 +828,7 @@ public class RegerPFloatTest { int cur_time = bit_width_segments[i][0]; int cur_value = bit_width_segments[i][1]; if (cur_time != pre_time && count_of_time != 0) { - run_length_time[pos_time][0] = count_of_time; - run_length_time[pos_time][1] = pre_time; + pos_time++; pre_time = cur_time; count_of_time = 1; @@ -1189,16 +836,14 @@ public class RegerPFloatTest { count_of_time++; pre_time = cur_time; if (count_of_time == 256) { - run_length_time[pos_time][0] = count_of_time; - run_length_time[pos_time][1] = pre_time; + pos_time++; count_of_time = 1; } } if (cur_value != pre_value && count_of_value != 0) { - run_length_value[pos_value][0] = count_of_value; - run_length_value[pos_value][1] = pre_value; + pos_value++; pre_value = cur_value; @@ -1207,21 +852,15 @@ public class RegerPFloatTest { count_of_value++; pre_value = cur_value; if (count_of_value == 256) { - run_length_value[pos_value][0] = count_of_value; - run_length_value[pos_value][1] = pre_value; pos_value++; count_of_value = 0; } } } if (count_of_time != 0) { - run_length_time[pos_time][0] = count_of_time; - run_length_time[pos_time][1] = pre_time; pos_time++; } if (count_of_value != 0) { - run_length_value[pos_value][0] = count_of_value; - run_length_value[pos_value][1] = pre_value; pos_value++; } @@ -1297,11 +936,7 @@ public class RegerPFloatTest { pos_encode += 2; intWord2Bytes(pos_value, pos_encode, encoded_result); pos_encode += 2; - // System.out.println("pos_time="+pos_time); - // System.out.println("pos_value="+pos_value); - // System.out.println(Arrays.deepToString(run_length_time)); - // System.out.println(Arrays.deepToString(run_length_value)); for (int i = 0; i < pos_time; i++) { int[] bit_width_time = run_length_time[i]; intByte2Bytes(bit_width_time[0], pos_encode, encoded_result); @@ -1309,8 +944,6 @@ public class RegerPFloatTest { intByte2Bytes(bit_width_time[1], pos_encode, encoded_result); pos_encode++; - // System.out.println("bit_width_time[0]="+bit_width_time[0]); - // System.out.println("bit_width_time[1]="+bit_width_time[1]); } for (int i = 0; i < pos_value; i++) { int[] bit_width_value = run_length_value[i]; @@ -1319,8 +952,6 @@ public class RegerPFloatTest { intByte2Bytes(bit_width_value[1], pos_encode, encoded_result); pos_encode++; - // System.out.println("bit_width_value[0]="+bit_width_value[0]); - // System.out.println("bit_width_value[1]="+bit_width_value[1]); } return pos_encode; @@ -1331,7 +962,6 @@ public class RegerPFloatTest { int[][] bit_width_segments, int[] raw_length, int segment_size, - int p, float[] theta, int pos_encode, byte[] encoded_result) { @@ -1357,12 +987,6 @@ public class RegerPFloatTest { int2Bytes(raw_length[4], pos_encode, encoded_result); pos_encode += 4; - // System.out.println(delta_segments[0][0]); - // System.out.println(delta_segments[0][1]); - // System.out.println(theta[0] + raw_length[3]); - // System.out.println(theta[1]); - // System.out.println(theta[2] + raw_length[4]); - // System.out.println(theta[3]); pos_encode = encodeRLEBitWidth2Bytes(bit_width_segments, pos_encode, encoded_result); for (int segment_i = 0; segment_i < segment_n; segment_i++) { @@ -1393,7 +1017,7 @@ public class RegerPFloatTest { public static int[][] getEncodeBitsRegressionNoTrain( - int[][] ts_block, int block_size, int[] raw_length, float[] theta, int segment_size, int p) { + int[][] ts_block, int block_size, int[] raw_length, float[] theta, int p) { int timestamp_delta_min = Integer.MAX_VALUE; int value_delta_min = Integer.MAX_VALUE; int[][] ts_block_delta = new int[ts_block.length][2]; @@ -1473,7 +1097,6 @@ public class RegerPFloatTest { int[][] bit_width_segments, int segment_size, int p) { - ArrayList<Byte> encoded_result = new ArrayList<>(); int block_size = delta_segments.length; int segment_n = (block_size - p) / segment_size; int result = 0; @@ -1543,8 +1166,8 @@ public class RegerPFloatTest { float epsilon_v = (float) ts_block[j][1] - theta[1]; for (int pi = 1; pi <= p; pi++) { - epsilon_r -= (float) (theta[2 * pi] * (float) ts_block[j - pi][0]); - epsilon_v -= (float) (theta[2 * pi + 1] * (float) ts_block[j - pi][1]); + epsilon_r -= theta[2 * pi] * (float) ts_block[j - pi][0]; + epsilon_v -= theta[2 * pi + 1] * (float) ts_block[j - pi][1]; } ts_block_delta[j][0] = (int) epsilon_r; ts_block_delta[j][1] = (int) epsilon_v; @@ -1582,7 +1205,6 @@ public class RegerPFloatTest { public static int[] getIStar( int[][] ts_block, ArrayList<Integer> min_index, - int block_size, int index, float[] theta, int p) { @@ -1606,7 +1228,7 @@ public class RegerPFloatTest { } public static int[] getIStar( - int[][] ts_block, ArrayList<Integer> min_index, int block_size, float[] theta, int p, int k) { + int[][] ts_block, ArrayList<Integer> min_index, float[] theta, int p) { int[] alpha_list = new int[4]; @@ -1647,7 +1269,6 @@ public class RegerPFloatTest { int supply_length, int[] third_value, int segment_size, - int k, int p, int encode_pos, byte[] cur_byte) { @@ -1659,11 +1280,8 @@ public class RegerPFloatTest { ts_block = new int[block_size][2]; ts_block_partition = new int[block_size][2]; for (int j = 0; j < block_size; j++) { -// data[j + i * block_size][0] -= min_time; ts_block[j][0] = data[j + i * block_size][0]-min_time; ts_block[j][1] = data[j + i * block_size][1]; - // ts_block_reorder[j][0] = data[j + i * block_size][0]; - // ts_block_reorder[j][1] = data[j + i * block_size][1]; } } else { ts_block = new int[supply_length][2]; @@ -1682,8 +1300,6 @@ public class RegerPFloatTest { } int2Bytes(min_time,encode_pos,cur_byte); encode_pos += 4; - // System.out.println(Arrays.deepToString(data)); - // System.out.println(Arrays.deepToString(ts_block)); int[] reorder_length = new int[5]; float[] theta_reorder = new float[2 * p + 2]; @@ -1693,28 +1309,16 @@ public class RegerPFloatTest { int[] raw_length = new int[5]; // length,max_bit_width_interval,max_bit_width_value,max_bit_width_deviation float[] theta = new float[2 * p + 2]; - int[][] ts_block_delta_reorder = new int[block_size][2]; - int[][] bit_width_segments_value; - int[][] ts_block_delta_time = new int[block_size][2]; - int[][] bit_width_segments_time; - int[][] ts_block_delta = new int[block_size][2]; - int[][] bit_width_segments_partition; + int[][] ts_block_delta_reorder; + int[][] ts_block_delta_time; + int[][] ts_block_delta; terminate(ts_block, theta_time, p); ts_block_delta_time = - getEncodeBitsRegressionNoTrain(ts_block, block_size, time_length, theta_time, segment_size, p); -// System.out.println(Arrays.deepToString(ts_block)); - bit_width_segments_time = segmentBitPacking(ts_block_delta_time, block_size, segment_size); -// time_length[0] = -// numberOfEncodeSegment2Bytes( -// ts_block_delta_time, -// bit_width_segments_time, -// segment_size, -// p -// ); -// System.out.println((Arrays.toString(theta_time))); -//// System.out.println((Arrays.deepToString(ts_block_delta))); -// System.out.println((Arrays.toString(time_length))); + getEncodeBitsRegressionNoTrain(ts_block, block_size, time_length, theta_time, p); + + segmentBitPacking(ts_block_delta_time, block_size, segment_size); + int pos_ts_block_partition = 0; if (third_value.length > 0) { @@ -1749,14 +1353,10 @@ public class RegerPFloatTest { terminate(ts_block_partition, theta, p); ts_block_delta = getEncodeBitsRegressionNoTrain( - ts_block_partition, block_size, raw_length, theta, segment_size, p); - bit_width_segments_partition = segmentBitPacking(ts_block_delta, block_size, segment_size); + ts_block_partition, block_size, raw_length, theta, p); + segmentBitPacking(ts_block_delta, block_size, segment_size); -// raw_length[0] = -// numberOfEncodeSegment2Bytes( -// ts_block_delta, bit_width_segments_partition, segment_size, p); - Arrays.sort( ts_block, (a, b) -> { @@ -1766,40 +1366,17 @@ public class RegerPFloatTest { terminate(ts_block, theta_reorder, p); ts_block_delta_reorder = getEncodeBitsRegressionNoTrain( - ts_block, block_size, reorder_length, theta_reorder, segment_size, p); - bit_width_segments_value = - segmentBitPacking(ts_block_delta_reorder, block_size, segment_size); -// reorder_length[0] = -// numberOfEncodeSegment2Bytes( -// ts_block_delta_reorder, -// bit_width_segments_value, -// segment_size, -// p -// ); - // System.out.println(Arrays.deepToString(ts_block_delta)); + ts_block, block_size, reorder_length, theta_reorder, p); + segmentBitPacking(ts_block_delta_reorder, block_size, segment_size); + int[] alpha_list; int choose = min3(time_length[0], raw_length[0], reorder_length[0]); ArrayList<Integer> min_index = new ArrayList<>(); int index_alpha_list = 0; -// if (time_length[0] < raw_length[0]) { -// raw_length = time_length.clone(); -//// System.out.println("time"); -// Arrays.sort( -// ts_block, -// (a, b) -> { -// if (a[0] == b[0]) return Integer.compare(a[1], b[1]); -// return Integer.compare(a[0], b[0]); -// }); -// theta = theta_time.clone(); -// } else { -// ts_block = ts_block_partition.clone(); -// -// } if (choose == 0) { raw_length = time_length.clone(); -// System.out.println("time"); Arrays.sort( ts_block, (a, b) -> { @@ -1809,10 +1386,9 @@ public class RegerPFloatTest { theta = theta_time.clone(); } else if (choose == 1) { ts_block = ts_block_partition.clone(); -// System.out.println("partition"); + } else { raw_length = reorder_length.clone(); -// System.out.println("value"); theta = theta_reorder.clone(); Arrays.sort( ts_block, @@ -1823,16 +1399,13 @@ public class RegerPFloatTest { index_alpha_list = 1; } -// System.out.println(Arrays.toString(theta)); -// System.out.println(Arrays.toString(raw_length)); - alpha_list = getIStar(ts_block, min_index, block_size, index_alpha_list, theta, p); + alpha_list = getIStar(ts_block, min_index, index_alpha_list, theta, p); int[] beta_list = new int[alpha_list.length]; int[][] new_length_list = new int[alpha_list.length][5]; int pos_new_length_list = 0; - // ArrayList<ArrayList<Integer>> new_length_list = new ArrayList<>(); for (int j = 0; j < alpha_list.length; j++) { int alpha = alpha_list[j]; @@ -1867,7 +1440,7 @@ public class RegerPFloatTest { beta_list[all_length.get(0).get(0)]); int[] new_length = new int[5]; ts_block_delta = - getEncodeBitsRegressionNoTrain(new_ts_block, block_size, new_length, theta, segment_size, p); + getEncodeBitsRegressionNoTrain(new_ts_block, block_size, new_length, theta, p); int[][] bit_width_segments = segmentBitPacking(ts_block_delta, block_size, segment_size); new_length[0] = numberOfEncodeSegment2Bytes( @@ -1883,7 +1456,7 @@ public class RegerPFloatTest { } else { break; } - alpha_list = getIStar(ts_block, min_index, block_size, theta, p, k); + alpha_list = getIStar(ts_block, min_index, theta, p); int alpha_size = alpha_list.length; for (int alpha_i = alpha_size - 1; alpha_i >= 0; alpha_i--) { @@ -1906,16 +1479,12 @@ public class RegerPFloatTest { isMoveable = isMovable(alpha_list, beta_list); } - // System.out.println("getEncodeBitsRegressionNoTrain before:" + - // Arrays.deepToString(ts_block_delta)); + ts_block_delta = - getEncodeBitsRegressionNoTrain(ts_block, block_size, raw_length, theta, segment_size, p); + getEncodeBitsRegressionNoTrain(ts_block, block_size, raw_length, theta, p); - // System.out.println("getEncodeBitsRegressionNoTrain after:" - // +Arrays.deepToString(ts_block_delta)); - // ArrayList<ArrayList<Integer>> bit_width_segments = new ArrayList<>(); int segment_n = (block_size - 1) / segment_size; int[][] bit_width_segments = new int[segment_n][2]; for (int segment_i = 0; segment_i < segment_n; segment_i++) { @@ -1939,8 +1508,7 @@ public class RegerPFloatTest { bit_width_segments[segment_i][1] = bit_width_value; } - // System.out.println(Arrays.deepToString(ts_block_delta)); - // System.out.println(Arrays.deepToString(bit_width_segments)); + encode_pos = encodeSegment2Bytes( @@ -1948,12 +1516,11 @@ public class RegerPFloatTest { bit_width_segments, raw_length, segment_size, - p, - theta, + theta, encode_pos, cur_byte); - // System.out.println("encode_pos="+encode_pos); + return encode_pos; } @@ -1963,9 +1530,8 @@ public class RegerPFloatTest { int[] third_value, int segment_size, int p, - int k, byte[] encoded_result) { -// for (int i = 0; i < p; i++) + block_size++; int length_all = data.length; @@ -1984,139 +1550,7 @@ public class RegerPFloatTest { intByte2Bytes(p, encode_pos, encoded_result); encode_pos += 1; - // ----------------------- compare data order by time, value and partition - // --------------------------- - int length_time = 0; - int length_value = 0; - int length_partition = 0; -// int[][] data_value = data.clone(); -// Arrays.sort( -// data_value, -// (a, b) -> { -// if (a[1] == b[1]) return Integer.compare(a[0], b[0]); -// return Integer.compare(a[1], b[1]); -// }); -// -// int[][] data_partition = new int[length_all][2]; -// int pos_data_partition = 0; -// -// for (int[] datum : data) { -// if (datum[1] > third_value[third_value.length - 1]) { -// data_partition[pos_data_partition][0] = datum[0]; -// data_partition[pos_data_partition][1] = datum[1]; -// pos_data_partition++; -// } -// } -// for (int third_i = third_value.length - 1; third_i > 0; third_i--) { -// for (int[] datum : data) { -// if (datum[1] <= third_value[third_i] && datum[1] > third_value[third_i - 1]) { -// data_partition[pos_data_partition][0] = datum[0]; -// data_partition[pos_data_partition][1] = datum[1]; -// pos_data_partition++; -// } -// } -// } -// for (int[] datum : data) { -// if (datum[1] <= third_value[0]) { -// data_partition[pos_data_partition][0] = datum[0]; -// data_partition[pos_data_partition][1] = datum[1]; -// pos_data_partition++; -// } -// } -// for (int i = 0; i < block_num; i++) { -// int[][] ts_block_time = new int[block_size][2]; -// int[][] ts_block_value = new int[block_size][2]; -// int[][] ts_block_partition = new int[block_size][2]; -// -// for (int j = 0; j < block_size; j++) { -// ts_block_time[j][0] = data[j + i * block_size][0]; -// ts_block_time[j][1] = data[j + i * block_size][1]; -// ts_block_value[j][0] = data_value[j + i * block_size][0]; -// ts_block_value[j][1] = data_value[j + i * block_size][1]; -// ts_block_partition[j][0] = data_partition[j + i * block_size][0]; -// ts_block_partition[j][1] = data_partition[j + i * block_size][1]; -// } -// -// int[] raw_length = new int[5]; -// float[] theta = new float[2 * p + 2]; -// int[][] ts_block_delta = -// getEncodeBitsRegression(ts_block_time, block_size, raw_length, theta, segment_size, p); -// int[][] bit_width_segments = segmentBitPacking(ts_block_delta, block_size, segment_size); -// length_time += -// numberOfEncodeSegment2Bytes( -// ts_block_delta, bit_width_segments, segment_size, p); -// -// int[] raw_length_value = new int[5]; -// float[] theta_value = new float[2 * p + 2]; -// int[][] ts_block_delta_value = -// getEncodeBitsRegression( -// ts_block_value, block_size, raw_length_value, theta_value, segment_size, p); -// int[][] bit_width_segments_value = -// segmentBitPacking(ts_block_delta_value, block_size, segment_size); -// length_value += -// numberOfEncodeSegment2Bytes( -// ts_block_delta_value, -// bit_width_segments_value, -// segment_size, -// p -// ); -// -// int[] raw_length_partition = new int[5]; -// float[] theta_partition = new float[2 * p + 2]; -// int[][] ts_block_delta_partition = -// getEncodeBitsRegression( -// ts_block_partition, -// block_size, -// raw_length_partition, -// theta_partition, -// segment_size, -// p); -// int[][] bit_width_segments_partition = -// segmentBitPacking(ts_block_delta_partition, block_size, segment_size); -// length_partition += -// numberOfEncodeSegment2Bytes( -// ts_block_delta_partition, -// bit_width_segments_partition, -// segment_size, -// p -// ); -// } - -// if (length_partition < length_time -// && length_partition < length_value) { // partition performs better -// data = data_partition; -// // for (int i = 0; i < 2; i++) { -// System.out.println("Partition"); -// for (int i = 0; i < block_num; i++) { -// encode_pos = -// REGERBlockEncoderPartition( -// data, i, block_size, segment_size, k, p, encode_pos, encoded_result); -// } -// } else { -// if (length_value < length_time) { // order by value performs better -// System.out.println("Value"); -// data = data_value; -//// for (int i = 0; i < 1; i++) { -// for (int i = 0; i < block_num; i++) { -// -// encode_pos = -// REGERBlockEncoder( -// data, -// 1, -// i, -// block_size, -// 0, -// third_value, -// segment_size, -// k, -// p, -// encode_pos, -// encoded_result); -// } -// } else { -// System.out.println("Time"); - // for (int i = 0; i < 2; i++) { -// for (int i = 0; i < 1; i++) { + for (int i = 0; i < block_num; i++) { encode_pos = REGERBlockEncoder( @@ -2126,13 +1560,11 @@ public class RegerPFloatTest { 0, third_value, segment_size, - k, - p, + p, encode_pos, encoded_result); } -// } -// } + int remaining_length = length_all - block_num * block_size; if (remaining_length == 1) { @@ -2163,8 +1595,7 @@ public class RegerPFloatTest { supple_length + remaining_length, third_value, segment_size, - k, - p, + p, encode_pos, encoded_result); } @@ -2172,256 +1603,6 @@ public class RegerPFloatTest { return encode_pos; } - public static double bytes2Double(ArrayList<Byte> encoded, int start, int num) { - if (num > 8) { - System.out.println("bytes2Doubleerror"); - return 0; - } - long value = 0; - for (int i = 0; i < 8; i++) { - value |= ((long) (encoded.get(i + start) & 0xff)) << (8 * i); - } - return Double.longBitsToDouble(value); - } - - public static float byte2float2(ArrayList<Byte> b, int index) { - int l; - l = b.get(index); - l &= 0xff; - l |= ((long) b.get(index + 1) << 8); - l &= 0xffff; - l |= ((long) b.get(index + 2) << 16); - l &= 0xffffff; - l |= ((long) b.get(index + 3) << 24); - return Float.intBitsToFloat(l); - } - - public static int bytes2Integer(ArrayList<Byte> encoded, int start, int num) { - int value = 0; - if (num > 4) { - System.out.println("bytes2Integer error"); - return 0; - } - for (int i = 0; i < num; i++) { - value <<= 8; - int b = encoded.get(i + start) & 0xFF; - value |= b; - } - return value; - } - - public static double[] dataStandardization(double[] array) { - StandardDeviation deviation = new StandardDeviation(); - for (int i = 0; i < array.length; i++) { - array[i] = array[i]; - } - return array; - } - - public static double[][] dataStandardizationDouble(double[][] arrays) { - double[][] result = new double[arrays[0].length][arrays.length]; - for (int i = 0; i < arrays.length; i++) { - double[] doubles = dataStandardization(arrays[i]); - for (int k = 0; k < result.length; k++) { - result[k][i] = doubles[k]; - } - } - return result; - } - -// @Test -// public void REGERPFloat() throws IOException { -// -// String parent_dir = -// "/Users/xiaojinzhao/Documents/GitHub/iotdb/iotdb-core/tsfile/src/test/resources/"; -// String output_parent_dir = -// "/Users/xiaojinzhao/Documents/GitHub/encoding-reorder/compression_ratio/p_float/"; -// String input_parent_dir = parent_dir + "trans_data/"; -// -// ArrayList<String> input_path_list = new ArrayList<>(); -// ArrayList<String> output_path_list = new ArrayList<>(); -// ArrayList<String> dataset_name = new ArrayList<>(); -// ArrayList<Integer> dataset_block_size = new ArrayList<>(); -// -// ArrayList<int[]> dataset_third = new ArrayList<>(); -// -// dataset_name.add("CS-Sensors"); -// dataset_name.add("Metro-Traffic"); -// dataset_name.add("USGS-Earthquakes"); -// dataset_name.add("YZ-Electricity"); -// dataset_name.add("GW-Magnetic"); -// dataset_name.add("TY-Fuel"); -// dataset_name.add("Cyber-Vehicle"); -// dataset_name.add("Vehicle-Charge"); -// dataset_name.add("Nifty-Stocks"); -// dataset_name.add("TH-Climate"); -// dataset_name.add("TY-Transport"); -// dataset_name.add("EPM-Education"); -// -// int[] dataset_0 = {547, 2816}; -// int[] dataset_1 = {1719, 3731}; -// int[] dataset_2 = {-48, -11, 6, 25, 52}; -// int[] dataset_3 = {8681, 13584}; -// int[] dataset_4 = {79, 184, 274}; -// int[] dataset_5 = {17, 68}; -// int[] dataset_6 = {677}; -// int[] dataset_7 = {1047, 1725}; -// int[] dataset_8 = {227, 499, 614, 1013}; -// int[] dataset_9 = {474, 678}; -// int[] dataset_10 = {4, 30, 38, 49, 58}; -// int[] dataset_11 = {5182, 8206}; -// -// dataset_third.add(dataset_0); -// dataset_third.add(dataset_1); -// dataset_third.add(dataset_2); -// dataset_third.add(dataset_3); -// dataset_third.add(dataset_4); -// dataset_third.add(dataset_5); -// dataset_third.add(dataset_6); -// dataset_third.add(dataset_7); -// dataset_third.add(dataset_8); -// dataset_third.add(dataset_9); -// dataset_third.add(dataset_10); -// dataset_third.add(dataset_11); -// -// for (int i = 0; i < dataset_name.size(); i++) { -// input_path_list.add(input_parent_dir + dataset_name.get(i)); -// dataset_block_size.add(1024); -// } -// -// output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0 -// // dataset_block_size.add(1024); -// -// output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv"); // 1 -// // dataset_block_size.add(512); -// output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv"); // 2 -// // dataset_block_size.add(512); -// output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3 -// // dataset_block_size.add(256); -// output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); // 4 -// // dataset_block_size.add(128); -// output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv"); // 5 -// // dataset_block_size.add(64); -// output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); // 6 -// // dataset_block_size.add(128); -// output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv"); // 7 -// // dataset_block_size.add(512); -// output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv"); // 8 -// // dataset_block_size.add(256); -// output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv"); // 9 -// // dataset_block_size.add(512); -// output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv"); // 10 -// // dataset_block_size.add(512); -// output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv"); // 11 -// // dataset_block_size.add(512); -// -// for (int file_i = 0; file_i < input_path_list.size(); file_i++) { -// // for (int file_i = 3; file_i < 4; file_i++) { -// -// String inputPath = input_path_list.get(file_i); -// String Output = output_path_list.get(file_i); -// System.out.println(inputPath); -// -// // speed -// int repeatTime = 1; // set repeat time -// -// File file = new File(inputPath); -// File[] tempList = file.listFiles(); -// -// CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); -// -// String[] head = { -// "Input Direction", -// "Encoding Algorithm", -// "Encoding Time", -// "Decoding Time", -// "Points", -// "p", -// "Compressed Size", -// "Compression Ratio" -// }; -// writer.writeRecord(head); // write header to output file -// -// assert tempList != null; -// -// for (File f : tempList) { -// System.out.println(f); -// for (int p = 2; p < 3; p++) { -// // for (int p = 1; p < 10; p++) { -// System.out.println("p=" + p); -// -// InputStream inputStream = Files.newInputStream(f.toPath()); -// CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8); -// ArrayList<ArrayList<Integer>> data = new ArrayList<>(); -// ArrayList<ArrayList<Integer>> data_decoded = new ArrayList<>(); -// -// // add a column to "data" -// loader.readHeaders(); -// data.clear(); -// while (loader.readRecord()) { -// ArrayList<Integer> tmp = new ArrayList<>(); -// tmp.add(Integer.valueOf(loader.getValues()[0])); -// tmp.add(Integer.valueOf(loader.getValues()[1])); -// data.add(tmp); -// } -// inputStream.close(); -// ArrayList<Integer> result2 = new ArrayList<>(); -// splitTimeStamp3(data, result2); -// int[][] data2_arr = new int[data.size()][2]; -// int min_time = data.get(0).get(0); -// for (int i = 0; i < data.size(); i++) { -// data2_arr[i][0] = data.get(i).get(0) - min_time; -// data2_arr[i][1] = data.get(i).get(1); -// } -// System.out.println(data2_arr[0][0]); -// byte[] encoded_result = new byte[data2_arr.length * 8]; -// -// long encodeTime = 0; -// long decodeTime = 0; -// double ratio = 0; -// double compressed_size = 0; -// int length = 0; -// -// long s = System.nanoTime(); -// -// for (int repeat_i = 0; repeat_i < 1; repeat_i++) -// length = -// ReorderingRegressionEncoder( -// data2_arr, -// dataset_block_size.get(file_i), -// dataset_third.get(file_i), -// 8, -// p, -// 1, -// encoded_result); -// -// long e = System.nanoTime(); -// encodeTime += ((e - s)); -// compressed_size += length; -// double ratioTmp = (double) compressed_size / (double) (data.size() * Integer.BYTES * 2); -// ratio += ratioTmp; -// s = System.nanoTime(); -// e = System.nanoTime(); -// decodeTime += ((e - s)); -// -// String[] record = { -// f.toString(), -// "REGER-32-FLOAT", -// String.valueOf(encodeTime), -// String.valueOf(decodeTime), -// String.valueOf(data.size()), -// String.valueOf(p), -// String.valueOf(compressed_size), -// String.valueOf(ratio) -// }; -// -// writer.writeRecord(record); -// System.out.println(ratio); -// } -// } -// writer.close(); -// } -// } @Test public void REGERPFloatVaryP() throws IOException { @@ -2485,52 +1666,33 @@ public class RegerPFloatTest { dataset_third.add(dataset_12); dataset_third.add(dataset_13); - for (int i = 0; i < dataset_name.size(); i++) { - input_path_list.add(input_parent_dir + dataset_name.get(i)); + for (String value : dataset_name) { + input_path_list.add(input_parent_dir + value); dataset_block_size.add(512); } output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0 - // dataset_block_size.add(1024); - - output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv"); // 1 - // dataset_block_size.add(512); - output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv"); // 2 - // dataset_block_size.add(512); + output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1 + output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2 output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3 - // dataset_block_size.add(256); - output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); // 4 - // dataset_block_size.add(128); - output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv"); // 5 - // dataset_block_size.add(64); - output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); // 6 - // dataset_block_size.add(128); - output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv"); // 7 - // dataset_block_size.add(512); - output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv"); // 8 - // dataset_block_size.add(256); - output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv"); // 9 - // dataset_block_size.add(512); - output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv"); // 10 - // dataset_block_size.add(512); - output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv"); // 11 - // dataset_block_size.add(512); + output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4 + output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5 + output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6 + output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7 + output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8 + output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9 + output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10 + output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11 output_path_list.add(output_parent_dir + "/FANYP-Sensors_ratio.csv"); // 12 output_path_list.add(output_parent_dir + "/TRAJET-Transport_ratio.csv"); // 13 - // 0 2 6 7 -// int[] file_lists = {1,9}; -// for (int file_i : file_lists) { for (int file_i = 0; file_i < input_path_list.size(); file_i++) { -// for (int file_i = 12; file_i < 14; file_i++) { -// + String inputPath = input_path_list.get(file_i); String Output = output_path_list.get(file_i); System.out.println(inputPath); - // speed - int repeatTime = 1; // set repeat time File file = new File(inputPath); File[] tempList = file.listFiles(); @@ -2556,18 +1718,16 @@ public class RegerPFloatTest { System.out.println(count_csv); count_csv ++; System.out.println(f); -// for (int p = 1; p < 2; p++) { + for (int p = 1; p < 10; p++) { System.out.println("p=" + p); InputStream inputStream = Files.newInputStream(f.toPath()); CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8); ArrayList<ArrayList<Integer>> data = new ArrayList<>(); - ArrayList<ArrayList<Integer>> data_decoded = new ArrayList<>(); // add a column to "data" loader.readHeaders(); - data.clear(); while (loader.readRecord()) { ArrayList<Integer> tmp = new ArrayList<>(); tmp.add(Integer.valueOf(loader.getValues()[0])); @@ -2583,7 +1743,7 @@ public class RegerPFloatTest { data2_arr[i][0] = data.get(i).get(0) - min_time; data2_arr[i][1] = data.get(i).get(1); } -// System.out.println(data2_arr[0][0]); + byte[] encoded_result = new byte[data2_arr.length * 8]; long encodeTime = 0; @@ -2602,13 +1762,12 @@ public class RegerPFloatTest { dataset_third.get(file_i), 16, p, - 1, - encoded_result); + encoded_result); long e = System.nanoTime(); encodeTime += ((e - s)); compressed_size += length; - double ratioTmp = (double) compressed_size / (double) (data.size() * Integer.BYTES * 2); + double ratioTmp = compressed_size / (double) (data.size() * Integer.BYTES * 2); ratio += ratioTmp; s = System.nanoTime(); e = System.nanoTime(); @@ -2628,7 +1787,7 @@ public class RegerPFloatTest { writer.writeRecord(record); System.out.println(ratio); } -// break; + } writer.close(); }
