This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch research/encoding-reorder in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 04e77c65728bc3cce200717baa41bfeec97c4e08 Author: xjz17 <[email protected]> AuthorDate: Mon Nov 27 16:26:26 2023 +0800 update exp --- .../iotdb/tsfile/encoding/REGERDoubleTest.java | 45 ++++++++-------------- ...RFloatTest.java => REGERFloatDecreaseTest.java} | 16 ++++---- .../iotdb/tsfile/encoding/REGERFloatTest.java | 4 +- .../iotdb/tsfile/encoding/RegerPDoubleTest.java | 21 ++++++++-- .../iotdb/tsfile/encoding/RegerPFloatTest.java | 23 ++++++++--- 5 files changed, 60 insertions(+), 49 deletions(-) diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERDoubleTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERDoubleTest.java index 2c4498f9c75..c4b5f3565e8 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERDoubleTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERDoubleTest.java @@ -3106,10 +3106,10 @@ public class REGERDoubleTest { output_path_list.add(output_parent_dir + "/FANYP-Sensors_ratio.csv"); // 12 output_path_list.add(output_parent_dir + "/TRAJET-Transport_ratio.csv"); // 13 -// int[] file_lists = {5,6,8,10}; -// for (int file_i : file_lists) { + int[] file_lists = {5,6,8,10}; + for (int file_i : file_lists) { // for (int file_i = 0; file_i < input_path_list.size(); file_i++) { - for (int file_i = 12; file_i < 14; file_i++) { +// for (int file_i = 12; file_i < 14; file_i++) { String inputPath = input_path_list.get(file_i); String Output = output_path_list.get(file_i); @@ -3138,7 +3138,7 @@ public class REGERDoubleTest { System.out.println(f); InputStream inputStream = Files.newInputStream(f.toPath()); CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8); -// ArrayList<ArrayList<Integer>> data = new ArrayList<>(); + ArrayList<Long> data = new ArrayList<>(); // add a column to "data" @@ -3147,39 +3147,24 @@ public class REGERDoubleTest { int time0 = Integer.parseInt(loader.getValues()[0]); int value0 = Integer.parseInt(loader.getValues()[1]); data.add(combine2Int(0, value0)); -// loader.readRecord(); -// int time1 = Integer.parseInt(loader.getValues()[0]); -// int value1 = Integer.parseInt(loader.getValues()[1]); -// System.out.println(time0); -// System.out.println(time1); -// System.out.println(value1); -// data.add(combine2Int(time1-time0, value1)); while (loader.readRecord()) { -// long time_tmp = Integer.parseInt(loader.getValues()[0])-time0; -// time_tmp <<=32; -// time_tmp += Integer.parseInt(loader.getValues()[1]); + int time_tmp = Integer.parseInt(loader.getValues()[0])-time0; -// System.out.println(time_tmp); int value_tmp = Integer.parseInt(loader.getValues()[1]); - data.add(combine2Int(time_tmp,value_tmp)); } -// System.out.println(getTime(data.get(1))); -// System.out.println((data.get(1))); -// System.out.println(data); inputStream.close(); ArrayList<Integer> result2 = new ArrayList<>(); splitTimeStamp3(data, result2); -// System.out.println(data); long[] data2_arr = new long[data.size()]; -// int min_time = data.get(0).get(0); + for (int i = 0; i < data.size(); i++) { data2_arr[i] = data.get(i) ; } -// System.out.println(data2_arr[0][0]); + byte[] encoded_result = new byte[data2_arr.length * 8]; long encodeTime = 0; long decodeTime = 0; @@ -3315,10 +3300,10 @@ public class REGERDoubleTest { output_path_list.add(output_parent_dir + "/FANYP-Sensors_ratio.csv"); // 12 output_path_list.add(output_parent_dir + "/TRAJET-Transport_ratio.csv"); // 13 -// int[] file_lists = {5,6,8,10}; -// for (int file_i : file_lists) { + int[] file_lists = {5,6,8,10}; + for (int file_i : file_lists) { // for (int file_i = 0; file_i < input_path_list.size(); file_i++) { - for (int file_i = 12; file_i < 14; file_i++) { +// for (int file_i = 12; file_i < 14; file_i++) { String inputPath = input_path_list.get(file_i); String Output = output_path_list.get(file_i); @@ -3393,7 +3378,7 @@ public class REGERDoubleTest { long decodeTime = 0; double ratio = 0; double compressed_size = 0; - int repeatTime2 = 10; + int repeatTime2 = 1; long s = System.nanoTime(); int length = 0; for (int repeat = 0; repeat < repeatTime2; repeat++) @@ -3526,10 +3511,10 @@ public class REGERDoubleTest { output_path_list.add(output_parent_dir + "/FANYP-Sensors_ratio.csv"); // 12 output_path_list.add(output_parent_dir + "/TRAJET-Transport_ratio.csv"); // 13 -// int[] file_lists = {5,6,8,10}; -// for (int file_i : file_lists) { + int[] file_lists = {5,6,8,10}; + for (int file_i : file_lists) { // for (int file_i = 0; file_i < input_path_list.size(); file_i++) { - for (int file_i = 12; file_i < 14; file_i++) { +// for (int file_i = 12; file_i < 14; file_i++) { String inputPath = input_path_list.get(file_i); String Output = output_path_list.get(file_i); @@ -3590,7 +3575,7 @@ public class REGERDoubleTest { long decodeTime = 0; double ratio = 0; double compressed_size = 0; - int repeatTime2 = 10; + int repeatTime2 = 1; long s = System.nanoTime(); int length = 0; for (int repeat = 0; repeat < repeatTime2; repeat++) diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatDecreaseTest.java similarity index 99% copy from iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java copy to iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatDecreaseTest.java index 59e1a109c24..2bbbb522e25 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatDecreaseTest.java @@ -15,7 +15,7 @@ import java.util.Objects; import static java.lang.Math.abs; -public class REGERFloatTest { +public class REGERFloatDecreaseTest { public static int getBitWith(int num) { if (num == 0) return 1; else return 32 - Integer.numberOfLeadingZeros(num); @@ -3140,7 +3140,7 @@ public class REGERFloatTest { output_path_list.add(output_parent_dir + "/FANYP-Sensors_ratio.csv"); // 12 output_path_list.add(output_parent_dir + "/TRAJET-Transport_ratio.csv"); // 13 - int[] file_lists = {5,6,8,10}; + int[] file_lists = {0}; for (int file_i : file_lists) { // for (int file_i = 0; file_i < input_path_list.size(); file_i++) { // for (int file_i = 12; file_i < 14; file_i++) { @@ -3244,8 +3244,8 @@ public class REGERFloatTest { } } - @Test - public void REGERVaryBlockSize() throws IOException { +// @Test + public static void REGERVaryBlockSize() throws IOException { // String parent_dir = "C:/Users/xiaoj/Desktop/test"; String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/iotdb/iotdb-core/tsfile/src/test/resources/"; @@ -3385,7 +3385,7 @@ public class REGERFloatTest { long decodeTime = 0; double ratio = 0; double compressed_size = 0; - int repeatTime2 = 10; + int repeatTime2 = 1; long s = System.nanoTime(); int length = 0; for (int repeat = 0; repeat < repeatTime2; repeat++) @@ -3424,8 +3424,8 @@ public class REGERFloatTest { } } - @Test - public void REGERVaryPackSize() throws IOException { +// @Test + public static void REGERVaryPackSize() throws IOException { // String parent_dir = "C:/Users/xiaoj/Desktop/test"; String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/iotdb/iotdb-core/tsfile/src/test/resources/"; @@ -3582,7 +3582,7 @@ public class REGERFloatTest { long decodeTime = 0; double ratio = 0; double compressed_size = 0; - int repeatTime2 = 10; + int repeatTime2 = 1; long s = System.nanoTime(); int length = 0; for (int repeat = 0; repeat < repeatTime2; repeat++) diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java index 59e1a109c24..1114b24d36f 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/REGERFloatTest.java @@ -3385,7 +3385,7 @@ public class REGERFloatTest { long decodeTime = 0; double ratio = 0; double compressed_size = 0; - int repeatTime2 = 10; + int repeatTime2 = 1; long s = System.nanoTime(); int length = 0; for (int repeat = 0; repeat < repeatTime2; repeat++) @@ -3582,7 +3582,7 @@ public class REGERFloatTest { long decodeTime = 0; double ratio = 0; double compressed_size = 0; - int repeatTime2 = 10; + int repeatTime2 = 1; long s = System.nanoTime(); int length = 0; for (int repeat = 0; repeat < repeatTime2; repeat++) diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPDoubleTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPDoubleTest.java index 1bb35b82a38..9731f616cb9 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPDoubleTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPDoubleTest.java @@ -742,7 +742,7 @@ public class RegerPDoubleTest { int[] b; int[][] new_length_list = new int[block_size][3]; int pos_new_length_list = 0; - int range = block_size / 16; + int range = block_size / 32; // alpha <= p if (alpha < p) { @@ -899,7 +899,7 @@ public class RegerPDoubleTest { int start_j = Math.max(alpha - range / 2, 1); int end_j = Math.min(alpha + range / 2, block_size - 1); - int j = 0; + int j = start_j; for (; j < alpha - p; j++) { b = adjustCase1(ts_block, alpha, j, theta, p); @@ -2098,6 +2098,9 @@ public class RegerPDoubleTest { dataset_name.add("TH-Climate"); dataset_name.add("TY-Transport"); dataset_name.add("EPM-Education"); + dataset_name.add("FANYP-Sensors"); + dataset_name.add("TRAJET-Transport"); + int[] dataset_0 = {547, 2816}; int[] dataset_1 = {1719, 3731}; @@ -2111,6 +2114,8 @@ public class RegerPDoubleTest { int[] dataset_9 = {474, 678}; int[] dataset_10 = {4, 30, 38, 49, 58}; int[] dataset_11 = {5182, 8206}; + int[] dataset_12 = {652477}; + int[] dataset_13 = {581388}; dataset_third.add(dataset_0); dataset_third.add(dataset_1); @@ -2124,6 +2129,8 @@ public class RegerPDoubleTest { dataset_third.add(dataset_9); dataset_third.add(dataset_10); dataset_third.add(dataset_11); + dataset_third.add(dataset_12); + dataset_third.add(dataset_13); for (int i = 0; i < dataset_name.size(); i++) { input_path_list.add(input_parent_dir + dataset_name.get(i)); @@ -2154,12 +2161,15 @@ public class RegerPDoubleTest { output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv"); // 10 // dataset_block_size.add(512); output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv"); // 11 + output_path_list.add(output_parent_dir + "/FANYP-Sensors_ratio.csv"); // 12 + output_path_list.add(output_parent_dir + "/TRAJET-Transport_ratio.csv"); // 13 + // dataset_block_size.add(512); // int[] file_lists = {0,2,6,7}; // // for (int file_i : file_lists) { - for (int file_i = 0; file_i < input_path_list.size(); file_i++) { - // for (int file_i = 0; file_i < 1; file_i++) { +// for (int file_i = 0; file_i < input_path_list.size(); file_i++) { + for (int file_i = 13; file_i < 14; file_i++) { String inputPath = input_path_list.get(file_i); String Output = output_path_list.get(file_i); @@ -2187,7 +2197,10 @@ public class RegerPDoubleTest { assert tempList != null; + int count_csv =0; for (File f : tempList) { + System.out.println(count_csv); + count_csv ++; System.out.println(f); // for (int p = 8; p < 9; p++) { for (int p = 1; p < 10; p++) { diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java index d8ffa99b07b..a9f762da8f9 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RegerPFloatTest.java @@ -730,7 +730,7 @@ public class RegerPFloatTest { int raw_value_delta_max_index = -1; int raw_abs_sum = raw_length[0]; - int range = block_size / 16; + int range = block_size / 32; ArrayList<Integer> j_star_list = new ArrayList<>(); // beta list of min b phi alpha to j @@ -2444,6 +2444,9 @@ public class RegerPFloatTest { dataset_name.add("TH-Climate"); dataset_name.add("TY-Transport"); dataset_name.add("EPM-Education"); + dataset_name.add("FANYP-Sensors"); + dataset_name.add("TRAJET-Transport"); + int[] dataset_0 = {547, 2816}; int[] dataset_1 = {1719, 3731}; @@ -2457,6 +2460,8 @@ public class RegerPFloatTest { int[] dataset_9 = {474, 678}; int[] dataset_10 = {4, 30, 38, 49, 58}; int[] dataset_11 = {5182, 8206}; + int[] dataset_12 = {652477}; + int[] dataset_13 = {581388}; dataset_third.add(dataset_0); dataset_third.add(dataset_1); @@ -2470,6 +2475,8 @@ public class RegerPFloatTest { dataset_third.add(dataset_9); dataset_third.add(dataset_10); dataset_third.add(dataset_11); + dataset_third.add(dataset_12); + dataset_third.add(dataset_13); for (int i = 0; i < dataset_name.size(); i++) { input_path_list.add(input_parent_dir + dataset_name.get(i)); @@ -2501,12 +2508,15 @@ public class RegerPFloatTest { // dataset_block_size.add(512); output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv"); // 11 // dataset_block_size.add(512); + output_path_list.add(output_parent_dir + "/FANYP-Sensors_ratio.csv"); // 12 + output_path_list.add(output_parent_dir + "/TRAJET-Transport_ratio.csv"); // 13 + // 0 2 6 7 -// int[] file_lists = {0,2,6,7}; -// for (int file_i : file_lists) { - for (int file_i = 0; file_i < input_path_list.size(); file_i++) { -// for (int file_i = 0; file_i < 1; file_i++) { + int[] file_lists = {13}; + for (int file_i : file_lists) { +// for (int file_i = 0; file_i < input_path_list.size(); file_i++) { +// for (int file_i = 12; file_i < 14; file_i++) { // String inputPath = input_path_list.get(file_i); String Output = output_path_list.get(file_i); @@ -2534,7 +2544,10 @@ public class RegerPFloatTest { assert tempList != null; + int count_csv = 0; for (File f : tempList) { + System.out.println(count_csv); + count_csv ++; System.out.println(f); // for (int p = 1; p < 2; p++) { for (int p = 1; p < 10; p++) {
