This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 100cf590344 Add a switch to control if allow null value be included in
the "quantity total" metric statistic (#16057)
100cf590344 is described below
commit 100cf590344146d7ec902e439f5e02843ed486e7
Author: libo <[email protected]>
AuthorDate: Wed Aug 13 17:35:04 2025 +0800
Add a switch to control if allow null value be included in the "quantity
total" metric statistic (#16057)
* Add a switch, control if allow null value be included in the "quantity
total" metric statistic.
* Consistent with the rules of insertRow function, are controlled by the
switch named "isEnableNullValueIncludedInQuatityStats".
* Consistent with the rules of insertRow function, are controlled by the
switch named "isEnableNullValueIncludedInQuatityStats".
* delete some unuseful variables.
* Correct the method that compute point numbers exist null value in the
Tablet structure.
* Support hot reload configuration with two kinds of command:
set configuration "enable_null_value_included_in_quatity_stats"="true" on 1;
load configuration;
Fix wrong about algorithm that compute table null value number.
Add a new configuration in the configuration template file.
* format test code.
* Rename name of config item.
* Format code.
---
.../iotdb/AlignedTimeseriesSessionExample.java | 78 ++++++++++++++++++++++
.../main/java/org/apache/iotdb/SessionExample.java | 55 +++++++++++++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 13 ++++
.../dataregion/memtable/AbstractMemTable.java | 56 ++++++++++++++--
.../conf/iotdb-system.properties.template | 5 ++
6 files changed, 211 insertions(+), 7 deletions(-)
diff --git
a/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java
b/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java
index 4dbcc16fd17..288bd65ddcb 100644
---
a/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java
+++
b/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java
@@ -623,4 +623,82 @@ public class AlignedTimeseriesSessionExample {
tablet3.reset();
}
}
+
+ private static void insertTabletsWithAlignedTimeseriesWithNullValue()
+ throws IoTDBConnectionException, StatementExecutionException {
+
+ List<IMeasurementSchema> schemaList1 = new ArrayList<>();
+ schemaList1.add(new MeasurementSchema("s1", TSDataType.INT64));
+ schemaList1.add(new MeasurementSchema("s2", TSDataType.INT64));
+
+ List<IMeasurementSchema> schemaList2 = new ArrayList<>();
+ schemaList2.add(new MeasurementSchema("s1", TSDataType.INT64));
+ schemaList2.add(new MeasurementSchema("s2", TSDataType.INT64));
+
+ List<IMeasurementSchema> schemaList3 = new ArrayList<>();
+ schemaList3.add(new MeasurementSchema("s1", TSDataType.INT64));
+ schemaList3.add(new MeasurementSchema("s2", TSDataType.INT64));
+
+ Tablet tablet1 = new Tablet(ROOT_SG2_D1_VECTOR6, schemaList1, 100);
+ Tablet tablet2 = new Tablet(ROOT_SG2_D1_VECTOR7, schemaList2, 100);
+ Tablet tablet3 = new Tablet(ROOT_SG2_D1_VECTOR8, schemaList3, 100);
+
+ Map<String, Tablet> tabletMap = new HashMap<>();
+ tabletMap.put(ROOT_SG2_D1_VECTOR6, tablet1);
+ tabletMap.put(ROOT_SG2_D1_VECTOR7, tablet2);
+ tabletMap.put(ROOT_SG2_D1_VECTOR8, tablet3);
+
+ // Method 1 to add tablet data
+ long timestamp = System.currentTimeMillis();
+ for (long row = 0; row < 90; row++) {
+ int row1 = tablet1.getRowSize();
+ int row2 = tablet2.getRowSize();
+ int row3 = tablet3.getRowSize();
+ tablet1.addTimestamp(row1, timestamp);
+ tablet2.addTimestamp(row2, timestamp);
+ tablet3.addTimestamp(row3, timestamp);
+ for (int i = 0; i < 2; i++) {
+ long value = new SecureRandom().nextLong();
+ tablet1.addValue(schemaList1.get(i).getMeasurementName(), row1, value);
+ tablet2.addValue(schemaList2.get(i).getMeasurementName(), row2, value);
+ tablet3.addValue(schemaList3.get(i).getMeasurementName(), row3, value);
+ }
+ if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) {
+ session.insertAlignedTablets(tabletMap, true);
+ tablet1.reset();
+ tablet2.reset();
+ tablet3.reset();
+ }
+ timestamp++;
+ }
+
+ for (long row = 90; row < 100; row++) {
+ int row1 = tablet1.getRowSize();
+ int row2 = tablet2.getRowSize();
+ int row3 = tablet3.getRowSize();
+ tablet1.addTimestamp(row1, timestamp);
+ tablet2.addTimestamp(row2, timestamp);
+ tablet3.addTimestamp(row3, timestamp);
+ for (int i = 0; i < 2; i++) {
+ Object value = null;
+ tablet1.addValue(schemaList1.get(i).getMeasurementName(), row1, value);
+ tablet2.addValue(schemaList2.get(i).getMeasurementName(), row2, value);
+ tablet3.addValue(schemaList3.get(i).getMeasurementName(), row3, value);
+ }
+ if (tablet1.getRowSize() == tablet1.getMaxRowNumber()) {
+ session.insertAlignedTablets(tabletMap, true);
+ tablet1.reset();
+ tablet2.reset();
+ tablet3.reset();
+ }
+ timestamp++;
+ }
+
+ if (tablet1.getRowSize() != 0) {
+ session.insertAlignedTablets(tabletMap, true);
+ tablet1.reset();
+ tablet2.reset();
+ tablet3.reset();
+ }
+ }
}
diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java
b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
index d5c6f0a5a6a..b7e9bddfe41 100644
--- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java
+++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
@@ -472,6 +472,61 @@ public class SessionExample {
}
}
+ private static void insertTabletWithNullValue()
+ throws IoTDBConnectionException, StatementExecutionException {
+ /*
+ * A Tablet example:
+ * device1
+ * time s1, s2, s3
+ * 1, 1, 1, 1
+ * 2, 2, 2, 2
+ * 3, 3, 3, 3
+ */
+ // The schema of measurements of one device
+ // only measurementId and data type in MeasurementSchema take effects in
Tablet
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("s1", TSDataType.INT64));
+ schemaList.add(new MeasurementSchema("s2", TSDataType.INT64));
+ schemaList.add(new MeasurementSchema("s3", TSDataType.INT64));
+
+ Tablet tablet = new Tablet(ROOT_SG1_D1, schemaList, 100);
+
+ long timestamp = System.currentTimeMillis();
+
+ for (long row = 0; row < 90; row++) {
+ int rowIndex = tablet.getRowSize();
+ tablet.addTimestamp(rowIndex, timestamp);
+ for (int s = 0; s < 3; s++) {
+ long value = random.nextLong();
+ tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex,
value);
+ }
+ if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
+ session.insertTablet(tablet, true);
+ tablet.reset();
+ }
+ timestamp++;
+ }
+
+ for (long row = 90; row < 100; row++) {
+ int rowIndex = tablet.getRowSize();
+ tablet.addTimestamp(rowIndex, timestamp);
+ for (int s = 0; s < 3; s++) {
+ Object value = null;
+ tablet.addValue(schemaList.get(s).getMeasurementName(), rowIndex,
value);
+ }
+ if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
+ session.insertTablet(tablet, true);
+ tablet.reset();
+ }
+ timestamp++;
+ }
+
+ if (tablet.getRowSize() != 0) {
+ session.insertTablet(tablet);
+ tablet.reset();
+ }
+ }
+
private static void insertTabletWithNullValues()
throws IoTDBConnectionException, StatementExecutionException {
/*
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 75dac521f2f..2f6accf13f7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -1164,6 +1164,8 @@ public class IoTDBConfig {
private long cacheLastValuesMemoryBudgetInByte = 4 * 1024 * 1024;
+ private boolean includeNullValueInWriteThroughputMetric = false;
+
IoTDBConfig() {}
public int getMaxLogEntriesNumPerBatch() {
@@ -4138,4 +4140,13 @@ public class IoTDBConfig {
public void setCacheLastValuesMemoryBudgetInByte(long
cacheLastValuesMemoryBudgetInByte) {
this.cacheLastValuesMemoryBudgetInByte = cacheLastValuesMemoryBudgetInByte;
}
+
+ public boolean isIncludeNullValueInWriteThroughputMetric() {
+ return includeNullValueInWriteThroughputMetric;
+ }
+
+ public void setIncludeNullValueInWriteThroughputMetric(
+ boolean includeNullValueInWriteThroughputMetric) {
+ this.includeNullValueInWriteThroughputMetric =
includeNullValueInWriteThroughputMetric;
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 7a6bfc8aa58..753607db4a6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1088,6 +1088,12 @@ public class IoTDBDescriptor {
loadQuerySampleThroughput(properties);
// update trusted_uri_pattern
loadTrustedUriPattern(properties);
+
+ conf.setIncludeNullValueInWriteThroughputMetric(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "include_null_value_in_write_throughput_metric",
+
String.valueOf(conf.isIncludeNullValueInWriteThroughputMetric()))));
}
private void loadSortBuffer(TrimProperties properties) {
@@ -2084,6 +2090,13 @@ public class IoTDBDescriptor {
// sort_buffer_size_in_bytes
loadSortBuffer(properties);
+
+ conf.setIncludeNullValueInWriteThroughputMetric(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "include_null_value_in_write_throughput_metric",
+ ConfigurationFileUtils.getConfigurationDefaultValue(
+ "include_null_value_in_write_throughput_metric"))));
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index a4b235650cf..f2f4d5e2943 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.IFullPath;
import org.apache.iotdb.commons.path.NonAlignedFullPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
@@ -211,7 +212,9 @@ public abstract class AbstractMemTable implements IMemTable
{
int pointsInserted =
insertRowNode.getMeasurements().length
- insertRowNode.getFailedMeasurementNumber()
- - nullPointsNumber;
+ -
(IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric()
+ ? 0
+ : nullPointsNumber);
totalPointsNum += pointsInserted;
return pointsInserted;
@@ -224,12 +227,16 @@ public abstract class AbstractMemTable implements
IMemTable {
Object[] values = insertRowNode.getValues();
List<IMeasurementSchema> schemaList = new ArrayList<>();
List<TSDataType> dataTypes = new ArrayList<>();
+ int nullPointsNumber = 0;
for (int i = 0; i < insertRowNode.getMeasurements().length; i++) {
// Use measurements[i] to ignore failed partial insert
if (measurements[i] == null
|| values[i] == null
|| insertRowNode.getColumnCategories() != null
&& insertRowNode.getColumnCategories()[i] !=
TsTableColumnCategory.FIELD) {
+ if (values[i] == null) {
+ nullPointsNumber++;
+ }
schemaList.add(null);
continue;
}
@@ -244,7 +251,11 @@ public abstract class AbstractMemTable implements
IMemTable {
MemUtils.getAlignedRowRecordSize(dataTypes, values,
insertRowNode.getColumnCategories());
writeAlignedRow(insertRowNode.getDeviceID(), schemaList,
insertRowNode.getTime(), values);
int pointsInserted =
- insertRowNode.getMeasurementColumnCnt() -
insertRowNode.getFailedMeasurementNumber();
+ insertRowNode.getMeasurementColumnCnt()
+ - insertRowNode.getFailedMeasurementNumber()
+ -
(IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric()
+ ? 0
+ : nullPointsNumber);
totalPointsNum += pointsInserted;
return pointsInserted;
}
@@ -253,11 +264,17 @@ public abstract class AbstractMemTable implements
IMemTable {
public int insertTablet(InsertTabletNode insertTabletNode, int start, int
end)
throws WriteProcessException {
try {
+ int nullPointsNumber = computeTabletNullPointsNumber(insertTabletNode,
start, end);
writeTabletNode(insertTabletNode, start, end);
memSize += MemUtils.getTabletSize(insertTabletNode, start, end);
int pointsInserted =
- (insertTabletNode.getDataTypes().length -
insertTabletNode.getFailedMeasurementNumber())
- * (end - start);
+ ((insertTabletNode.getDataTypes().length -
insertTabletNode.getFailedMeasurementNumber())
+ * (end - start))
+ - (IoTDBDescriptor.getInstance()
+ .getConfig()
+ .isIncludeNullValueInWriteThroughputMetric()
+ ? 0
+ : nullPointsNumber);
totalPointsNum += pointsInserted;
return pointsInserted;
} catch (RuntimeException e) {
@@ -270,13 +287,19 @@ public abstract class AbstractMemTable implements
IMemTable {
InsertTabletNode insertTabletNode, int start, int end, TSStatus[]
results)
throws WriteProcessException {
try {
+ int nullPointsNumber = computeTabletNullPointsNumber(insertTabletNode,
start, end);
writeAlignedTablet(insertTabletNode, start, end, results);
// TODO-Table: what is the relation between this and
TsFileProcessor.checkMemCost
memSize += MemUtils.getAlignedTabletSize(insertTabletNode, start, end,
results);
int pointsInserted =
- (insertTabletNode.getMeasurementColumnCnt()
- - insertTabletNode.getFailedMeasurementNumber())
- * (end - start);
+ ((insertTabletNode.getMeasurementColumnCnt()
+ - insertTabletNode.getFailedMeasurementNumber())
+ * (end - start))
+ - (IoTDBDescriptor.getInstance()
+ .getConfig()
+ .isIncludeNullValueInWriteThroughputMetric()
+ ? 0
+ : nullPointsNumber);
totalPointsNum += pointsInserted;
return pointsInserted;
} catch (RuntimeException e) {
@@ -284,6 +307,25 @@ public abstract class AbstractMemTable implements
IMemTable {
}
}
+ private static int computeTabletNullPointsNumber(
+ InsertTabletNode insertTabletNode, int start, int end) {
+ Object[] values = insertTabletNode.getBitMaps();
+ int nullPointsNumber = 0;
+ if (values != null) {
+ for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) {
+ BitMap bitMap = (BitMap) values[i];
+ if (bitMap != null && !bitMap.isAllUnmarked()) {
+ for (int j = start; j < end; j++) {
+ if (bitMap.isMarked(j)) {
+ nullPointsNumber++;
+ }
+ }
+ }
+ }
+ }
+ return nullPointsNumber;
+ }
+
@Override
public void write(
IDeviceID deviceId,
diff --git
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 7eb20db14f3..288c814613e 100644
---
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -2154,3 +2154,8 @@
write_request_remote_dispatch_max_retry_duration_in_ms=60000
# effectiveMode: hot_reload
# Datatype: boolean
enable_retry_for_unknown_error=false
+
+# The switch to control if allow null value be included in the "quantity
total" metric statistic.
+# effectiveMode: hot_reload
+# Datatype: Boolean
+include_null_value_in_write_throughput_metric=false
\ No newline at end of file
