This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/research/M4-visualization by
this push:
new 469918bdfa 28%
469918bdfa is described below
commit 469918bdfa3614e263ee9b089e0697b1ee21202e
Author: Lei Rui <[email protected]>
AuthorDate: Tue Oct 11 21:45:47 2022 +0800
28%
---
.../session/MyRealDataTest1_WriteAndQuery.java | 12 +--
.../iotdb/session/MyRealDataTest2_OnlyQuery.java | 6 +-
.../session/MyRealDataTest3_WriteAndRawQuery.java | 6 +-
.../session/MySmallRealDataWriteQueryTest.java | 39 ++++----
.../encoding/decoder/DeltaBinaryDecoder.java | 13 ++-
.../encoding/encoder/DeltaBinaryEncoder.java | 22 +++--
.../iotdb/tsfile/read/reader/page/PageReader.java | 103 ++++-----------------
7 files changed, 74 insertions(+), 127 deletions(-)
diff --git
a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
index aefbdef833..bff308ff23 100644
---
a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
+++
b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
@@ -57,7 +57,7 @@ public class MyRealDataTest1_WriteAndQuery {
// private static long dataMaxTime = 617426057626L;
// private static long total_time_length = dataMaxTime - dataMinTime;
// private static int total_point_number = 1200000;
- // private static int iotdb_chunk_point_size = 100000;
+ // private static int iotdb_chunk_point_size = 100;
// private static long chunkAvgTimeLen = (long) Math
// .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
// private static String filePath =
@@ -82,7 +82,7 @@ public class MyRealDataTest1_WriteAndQuery {
// private static long dataMaxTime = 1329965999991045200L;
// private static long total_time_length = dataMaxTime - dataMinTime;
// private static int total_point_number = 1076102;
- // private static int iotdb_chunk_point_size = 100000;
+ // private static int iotdb_chunk_point_size = 100;
// private static long chunkAvgTimeLen = (long) Math
// .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
// private static String filePath =
@@ -106,7 +106,7 @@ public class MyRealDataTest1_WriteAndQuery {
// private static long dataMaxTime = 1329965999991000000L;
// private static long total_time_length = dataMaxTime - dataMinTime;
// private static int total_point_number = 1076102;
- // private static int iotdb_chunk_point_size = 100000;
+ // private static int iotdb_chunk_point_size = 100;
// private static long chunkAvgTimeLen =
// (long)
// Math.ceil(
@@ -132,7 +132,7 @@ public class MyRealDataTest1_WriteAndQuery {
// private static long dataMaxTime = 1627380839563L;
// private static long total_time_length = dataMaxTime - dataMinTime;
// private static int total_point_number = 1943180;
- // private static int iotdb_chunk_point_size = 100000;
+ // private static int iotdb_chunk_point_size = 100;
// private static long chunkAvgTimeLen =
// (long)
// Math.ceil(
@@ -159,7 +159,7 @@ public class MyRealDataTest1_WriteAndQuery {
private static long dataMaxTime = 1627380839000L;
private static long total_time_length = dataMaxTime - dataMinTime;
private static int total_point_number = 1860287;
- private static int iotdb_chunk_point_size = 100000;
+ private static int iotdb_chunk_point_size = 100;
private static long chunkAvgTimeLen =
(long)
Math.ceil(
@@ -184,7 +184,7 @@ public class MyRealDataTest1_WriteAndQuery {
// private static long dataMaxTime = 1642656230000L;
// private static long total_time_length = dataMaxTime - dataMinTime;
// private static int total_point_number = 1330764;
- // private static int iotdb_chunk_point_size = 100000;
+ // private static int iotdb_chunk_point_size = 100;
// private static long chunkAvgTimeLen =
// (long)
// Math.ceil(
diff --git
a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest2_OnlyQuery.java
b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest2_OnlyQuery.java
index a2df4fdc9b..e99a27e4fd 100644
---
a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest2_OnlyQuery.java
+++
b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest2_OnlyQuery.java
@@ -57,7 +57,7 @@ public class MyRealDataTest2_OnlyQuery {
// private static long dataMaxTime = 617426057626L;
// private static long total_time_length = dataMaxTime - dataMinTime;
// private static int total_point_number = 1200000;
- // private static int iotdb_chunk_point_size = 100000;
+ // private static int iotdb_chunk_point_size = 100;
// private static long chunkAvgTimeLen = (long) Math
// .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
// private static String filePath =
@@ -81,7 +81,7 @@ public class MyRealDataTest2_OnlyQuery {
// private static long dataMaxTime = 1329965999991045200L;
// private static long total_time_length = dataMaxTime - dataMinTime;
// private static int total_point_number = 1076102;
- // private static int iotdb_chunk_point_size = 100000;
+ // private static int iotdb_chunk_point_size = 100;
// private static long chunkAvgTimeLen = (long) Math
// .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
// private static String filePath =
@@ -105,7 +105,7 @@ public class MyRealDataTest2_OnlyQuery {
private static long dataMaxTime = 1627380839563L;
private static long total_time_length = dataMaxTime - dataMinTime;
private static int total_point_number = 1943180;
- private static int iotdb_chunk_point_size = 100000;
+ private static int iotdb_chunk_point_size = 100;
private static long chunkAvgTimeLen =
(long)
Math.ceil(
diff --git
a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest3_WriteAndRawQuery.java
b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest3_WriteAndRawQuery.java
index 7483330705..e647d0f8ad 100644
---
a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest3_WriteAndRawQuery.java
+++
b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest3_WriteAndRawQuery.java
@@ -61,7 +61,7 @@ public class MyRealDataTest3_WriteAndRawQuery {
private static long dataMaxTime = 617426057626L;
private static long total_time_length = dataMaxTime - dataMinTime;
private static int total_point_number = 1200000;
- private static int iotdb_chunk_point_size = 100000;
+ private static int iotdb_chunk_point_size = 100;
private static long chunkAvgTimeLen =
(long)
Math.ceil(
@@ -86,7 +86,7 @@ public class MyRealDataTest3_WriteAndRawQuery {
// private static long dataMaxTime = 1329965999991045200L;
// private static long total_time_length = dataMaxTime - dataMinTime;
// private static int total_point_number = 1076102;
- // private static int iotdb_chunk_point_size = 100000;
+ // private static int iotdb_chunk_point_size = 100;
// private static long chunkAvgTimeLen = (long) Math
// .ceil(total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
// private static String filePath =
@@ -110,7 +110,7 @@ public class MyRealDataTest3_WriteAndRawQuery {
// private static long dataMaxTime = 1627380839563L;
// private static long total_time_length = dataMaxTime - dataMinTime;
// private static int total_point_number = 1943180;
- // private static int iotdb_chunk_point_size = 100000;
+ // private static int iotdb_chunk_point_size = 100;
// private static long chunkAvgTimeLen =
// (long)
// Math.ceil(
diff --git
a/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
b/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
index 61ce07c88f..c4c49598a2 100644
---
a/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
+++
b/session/src/test/java/org/apache/iotdb/session/MySmallRealDataWriteQueryTest.java
@@ -1,5 +1,13 @@
package org.apache.iotdb.session;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
@@ -11,20 +19,11 @@ import org.apache.iotdb.session.SessionDataSet.DataIterator;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
public class MySmallRealDataWriteQueryTest {
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
@@ -50,23 +49,25 @@ public class MySmallRealDataWriteQueryTest {
private static long dataMaxTime = 25599285703L;
private static long total_time_length = dataMaxTime - dataMinTime;
private static int total_point_number = 50000;
- private static int iotdb_chunk_point_size = 10000;
+ private static int iotdb_chunk_point_size = 100;// must be smaller than
BLOCK_DEFAULT_SIZE = 128 to fulfill the assumption that page=pack
private static long chunkAvgTimeLen =
(long)
Math.ceil(
total_time_length / Math.ceil(total_point_number * 1.0 /
iotdb_chunk_point_size));
private static String filePath =
"D:\\github\\m4-lsm\\iotdb\\session\\src\\test\\java\\org\\apache\\iotdb\\session\\BallSpeedSmallData.csv";
- private static int deletePercentage = 0; // 0 means no deletes. 0-100
- private static int deleteLenPercentage = 0; // 0-100
每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ private static int deletePercentage = 1; // 0 means no deletes. 0-100
+ private static int deleteLenPercentage = 30; // 0-100
每次删除的时间长度，用chunkAvgTimeLen的百分比表示
private static int timeIdx = 0; // 时间戳idx，从0开始
private static int valueIdx = 1; // 值idx，从0开始
private static int w = 3;
private static long range = total_time_length;
private static boolean enableRegularityTimeDecode = false;
private static long regularTimeInterval = 511996L;
- // private static long regularTimeInterval = 511997L;
- private static String approach = "cpv"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+ private static String approach = "mac"; // 选择查询执行算法: 1: MAC, (2: MOC,) 3: CPV
+
+ private static long seed = 20221011;
+ private static Random random = new Random(seed);
@Before
public void setUp() throws Exception {
@@ -113,7 +114,9 @@ public class MySmallRealDataWriteQueryTest {
EnvironmentUtils.cleanEnv();
}
- /** Before writing data, make sure check the server parameter
configurations. */
+ /**
+ * Before writing data, make sure check the server parameter configurations.
+ */
// Usage: java -jar WriteData-0.12.4.jar device measurement dataType
timestamp_precision
// total_time_length total_point_number iotdb_chunk_point_size filePath
deleteFreq deleteLen
// timeIdx valueIdx
@@ -149,7 +152,8 @@ public class MySmallRealDataWriteQueryTest {
} else {
// randomize between [dataMinTime, dataMaxTime-range]
minTime =
- (long) Math.ceil(dataMinTime + Math.random() * (dataMaxTime - range
- dataMinTime + 1));
+ (long) Math
+ .ceil(dataMinTime + random.nextDouble() * (dataMaxTime - range -
dataMinTime + 1));
interval = (long) Math.ceil((double) range / w);
}
maxTime = minTime + interval * w;
@@ -311,7 +315,8 @@ public class MySmallRealDataWriteQueryTest {
long deleteStartTime =
(long)
Math.ceil(
- lastDeleteMinTime + Math.random() * (rightBound -
lastDeleteMinTime + 1));
+ lastDeleteMinTime + random.nextDouble() * (rightBound -
lastDeleteMinTime
+ + 1));
long deleteEndTime = deleteStartTime + deleteLen - 1;
session.deleteData(deletePaths, deleteStartTime, deleteEndTime);
System.out.println("[[[[delete]]]]]" + deleteStartTime + "," +
deleteEndTime);
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index cf80fdb01f..bd55464282 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -173,7 +173,8 @@ public abstract class DeltaBinaryDecoder extends Decoder {
public static class LongDeltaDecoder extends DeltaBinaryDecoder {
private long firstValue;
- private long[] data;
+ private long[] data; // NOTE this does not include firstValue
+ private long[] allData; // assuming only one pack in the buffer to be
decoded
private long previous;
/**
* minimum value for all difference.
@@ -201,11 +202,17 @@ public abstract class DeltaBinaryDecoder extends Decoder {
public long[] getDataArray4CPV(ByteBuffer buffer) {
// assuming only one pack in the buffer to be decoded
if (isDataReady) {
- return data;
+ return allData;
}
+
loadIntBatch(buffer);
+ allData = new long[packNum + 1];
+ allData[0] = firstValue;
+ if (packNum >= 0) {
+ System.arraycopy(data, 0, allData, 1, packNum);
+ }
isDataReady = true;
- return data;
+ return allData;
}
// /**
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
index f8fecf4454..05f8319915 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
@@ -19,17 +19,15 @@
package org.apache.iotdb.tsfile.encoding.encoder;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
/**
* DeltaBinaryEncoder is a encoder for compressing data in type of integer and
long. We adapt a
* hypothesis that contiguous data points have similar values. Thus the
difference value of two
@@ -38,8 +36,8 @@ import java.io.IOException;
* Given an array which length is n, if all values in input data array are all
positive and less
* than 2^m, we need actually m*n, but not 32*n bits to store the array.
*
- * <p>DeltaBinaryEncoder calculates difference between two adjacent points and
record the minimum of
- * those difference values firstly. Then it saves two_diff value that
difference minus minimum of
+ * <p>DeltaBinaryEncoder calculates difference between two adjacent points and
record the minimum
+ * of those difference values firstly. Then it saves two_diff value that
difference minus minimum of
* them, to make sure all two_diff values are positive. Then it statistics the
longest bit length
* {@code m} it takes for each two_diff value, which means the bit length that
maximum two_diff
* value takes. Only the low m bits are saved into result byte array for all
two_diff values.
@@ -78,7 +76,9 @@ public abstract class DeltaBinaryEncoder extends Encoder {
protected abstract int calculateBitWidthsForDeltaBlockBuffer();
- /** write all data into {@code encodingBlockBuffer}. */
+ /**
+ * write all data into {@code encodingBlockBuffer}.
+ */
private void writeDataWithMinWidth() {
for (int i = 0; i < writeIndex; i++) {
writeValueToBytes(i);
@@ -111,7 +111,9 @@ public abstract class DeltaBinaryEncoder extends Encoder {
writeIndex = -1;
}
- /** calling this method to flush all values which haven't encoded to result
byte array. */
+ /**
+ * calling this method to flush all values which haven't encoded to result
byte array.
+ */
@Override
public void flush(ByteArrayOutputStream out) {
try {
@@ -172,7 +174,7 @@ public abstract class DeltaBinaryEncoder extends Encoder {
* input a integer.
*
* @param value value to encode
- * @param out the ByteArrayOutputStream which data encode into
+ * @param out the ByteArrayOutputStream which data encode into
*/
public void encodeValue(int value, ByteArrayOutputStream out) {
if (writeIndex == -1) {
@@ -380,7 +382,7 @@ public abstract class DeltaBinaryEncoder extends Encoder {
* input a integer or long value.
*
* @param value value to encode
- * @param out - the ByteArrayOutputStream which data encode into
+ * @param out - the ByteArrayOutputStream which data encode into
*/
public void encodeValue(long value, ByteArrayOutputStream out) {
if (writeIndex == -1) {
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index d6629b40cb..71fec969c0 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -59,7 +59,7 @@ public class PageReader implements IPageReader {
/**
* time column in memory
*/
- public ByteBuffer timeBuffer;
+ public ByteBuffer timeBuffer; // solely consumed either by
getDataArray4CPV() or by
timeDecoder.hasNext(timeBuffer)&timeDecoder.readLong(timeBuffer)
/**
* value column in memory
@@ -360,102 +360,35 @@ public class PageReader implements IPageReader {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
@Override
public BatchData getAllSatisfiedPageData(boolean ascending) throws
IOException {
- // TODO: return null(value no need) or FP&LP
BatchData pageData = BatchDataFactory.createBatchData(dataType, ascending,
false);
- while (timeDecoder.hasNext(timeBuffer)) { // TODO: timeDecoder.data
- long timestamp = timeDecoder.readLong(timeBuffer);
+ long[] timeData = ((LongDeltaDecoder)
timeDecoder).getDataArray4CPV(timeBuffer);
+ for (long timestamp : timeData) {
+ // TODO delay the decode of value until the timestamp is valid, skip to
the next point when t is invalid
long aLong = valueDecoder
.readLong(valueBuffer); // hard-coded, assuming value is long data
type
- if (!isDeleted(timestamp) && (filter == null ||
filter.satisfy(timestamp, aLong))) {
+
+ if (!isDeleted(timestamp) && (filter == null ||
filter.satisfy(timestamp, null))) {
+ // cannot remove filter here because M4-UDF uses time filters, but we
can delay the use of value object
+ // assuming the filter is always timeFilter
+
pageData.putLong(timestamp, aLong);
}
+ }
-// switch (dataType) {
-// case BOOLEAN:
-// boolean aBoolean = valueDecoder.readBoolean(valueBuffer);
-// if (!isDeleted(timestamp)
-// && (filter == null || filter.satisfy(timestamp, aBoolean))) {
// TODO:remove
-// pageData.putBoolean(timestamp, aBoolean);
-// }
-// break;
-// case INT32:
-// int anInt = valueDecoder.readInt(valueBuffer);
-// if (!isDeleted(timestamp) && (filter == null ||
filter.satisfy(timestamp, anInt))) {
-// pageData.putInt(timestamp, anInt);
-// }
-// break;
-// case INT64:
-// long aLong = valueDecoder.readLong(valueBuffer);
-// if (!isDeleted(timestamp) && (filter == null ||
filter.satisfy(timestamp, aLong))) {
-// pageData.putLong(timestamp, aLong);
-// }
-// break;
-// case FLOAT:
-// float aFloat = valueDecoder.readFloat(valueBuffer);
-// if (!isDeleted(timestamp) && (filter == null ||
filter.satisfy(timestamp, aFloat))) {
-// pageData.putFloat(timestamp, aFloat);
-// }
-// break;
-// case DOUBLE:
-// double aDouble = valueDecoder.readDouble(valueBuffer);
-// if (!isDeleted(timestamp) && (filter == null ||
filter.satisfy(timestamp, aDouble))) {
-// pageData.putDouble(timestamp, aDouble);
-// }
-// break;
-// case TEXT:
-// Binary aBinary = valueDecoder.readBinary(valueBuffer);
-// if (!isDeleted(timestamp) && (filter == null ||
filter.satisfy(timestamp, aBinary))) {
-// pageData.putBinary(timestamp, aBinary);
-// }
-// break;
-// default:
-// throw new UnSupportedDataTypeException(String.valueOf(dataType));
+// while (timeDecoder.hasNext(timeBuffer)) { // TODO: timeDecoder.data
+// long timestamp = timeDecoder.readLong(timeBuffer);
+// long aLong = valueDecoder
+// .readLong(valueBuffer); // hard-coded, assuming value is long data
type
+// if (!isDeleted(timestamp) && (filter == null ||
filter.satisfy(timestamp, aLong))) {
+// // cannot remove filter here because M4-UDF uses time filters
+// pageData.putLong(timestamp, aLong);
// }
+// }
- }
return pageData.flip();
}
-// public BatchData getAllSatisfiedPageData_new(boolean ascending) throws
IOException {
-// // TODO: return null(value no need) or FP&LP
-//
-// BatchData pageData = BatchDataFactory.createBatchData(dataType,
ascending, false);
-//
-// // originally there are chunk->page->pack, but here we write hard coded
assuming chunk=page=pack
-// // therefore we get the data array directly in batch, instead of
point-by-point
-// // Actually, originally decoder->data array->pageReader->batchData,
iterating these points twice.
-// // Here, we try to merge into once.
-// long[] timestamps =
((LongDeltaDecoder)timeDecoder).getDataArray4CPV(timeBuffer);
-//
-// // here we write hard-coded assuming always long data type
-//// long aLong = valueDecoder.readLong(valueBuffer);
-// // TODO
-//
-//
-// // here we write hard-coded assuming no filters
-// if (!isDeleted(timestamp)) {
-// pageData.putLong(timestamp, aLong);
-// }
-//
-//// }
-//
-//// while (timeDecoder.hasNext(timeBuffer)) { // TODO: timeDecoder.data
-////
-//// long timestamp = timeDecoder.readLong(timeBuffer);
-////
-//// // here we write hard-coded assuming always long data type
-//// long aLong = valueDecoder.readLong(valueBuffer);
-////
-//// // here we write hard-coded assuming no filters
-//// if (!isDeleted(timestamp)) {
-//// pageData.putLong(timestamp, aLong);
-//// }
-////
-//// }
-// return pageData.flip();
-// }
-
@Override
public Statistics getStatistics() {
return pageHeader.getStatistics();
