This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new dacbf9268db Optimize InsertTabletStatement tablet conversion (#17805)
dacbf9268db is described below
commit dacbf9268db160666a3b3a3a8e31c53eb88c8f27
Author: Caideyipi <[email protected]>
AuthorDate: Wed Jun 3 18:57:20 2026 +0800
Optimize InsertTabletStatement tablet conversion (#17805)
---
.../plan/statement/crud/InsertTabletStatement.java | 118 ++++++++++++---------
1 file changed, 66 insertions(+), 52 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
index 82f5990e037..8ce0d534a49 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
@@ -740,25 +740,38 @@ public class InsertTabletStatement extends
InsertBaseStatement implements ISchem
// Build schemas and track valid column indices (skip null columns)
// measurements and dataTypes being null is standard - skip those columns
- final List<IMeasurementSchema> schemas = new ArrayList<>();
- final List<Integer> validColumnIndices = new ArrayList<>();
- for (int i = 0; i < originalSchemaSize; i++) {
- if (dataTypes != null && measurements[i] != null && dataTypes[i] !=
null) {
- // Create MeasurementSchema if not present
- schemas.add(new MeasurementSchema(measurements[i], dataTypes[i]));
- validColumnIndices.add(i);
+ final List<IMeasurementSchema> schemas = new
ArrayList<>(originalSchemaSize);
+ final int[] validColumnIndices = new int[originalSchemaSize];
+ int validColumnCount = 0;
+ if (dataTypes != null) {
+ final int dataTypeSize = Math.min(originalSchemaSize,
dataTypes.length);
+ for (int i = 0; i < dataTypeSize; i++) {
+ if (measurements[i] != null && dataTypes[i] != null) {
+ final MeasurementSchema measurementSchema =
+ measurementSchemas != null && i < measurementSchemas.length
+ ? measurementSchemas[i]
+ : null;
+ schemas.add(
+ measurementSchema != null
+ &&
Objects.equals(measurementSchema.getMeasurementName(), measurements[i])
+ && measurementSchema.getType() == dataTypes[i]
+ ? measurementSchema
+ : new MeasurementSchema(measurements[i], dataTypes[i]));
+ validColumnIndices[validColumnCount++] = i;
+ }
+ // Skip null columns - don't add to schemas or validColumnIndices
}
- // Skip null columns - don't add to schemas or validColumnIndices
}
- final int schemaSize = schemas.size();
+ final int schemaSize = validColumnCount;
// Get columnTypes (for table model) - only for valid columns
final TsTableColumnCategory[] columnCategories =
this.getColumnCategories();
- final List<ColumnCategory> tabletColumnTypes = new ArrayList<>();
+ final List<ColumnCategory> tabletColumnTypes = new
ArrayList<>(schemaSize);
if (columnCategories != null && columnCategories.length > 0) {
- for (final int validIndex : validColumnIndices) {
- if (columnCategories[validIndex] != null) {
+ for (int i = 0; i < schemaSize; i++) {
+ final int validIndex = validColumnIndices[i];
+ if (validIndex < columnCategories.length &&
columnCategories[validIndex] != null) {
tabletColumnTypes.add(columnCategories[validIndex].toTsFileColumnType());
} else {
tabletColumnTypes.add(ColumnCategory.FIELD);
@@ -775,9 +788,10 @@ public class InsertTabletStatement extends
InsertBaseStatement implements ISchem
final long[] times = this.getTimes();
final int rowSize = this.getRowCount();
final long[] timestamps;
- if (times != null && times.length >= rowSize && rowSize > 0) {
- timestamps = new long[rowSize];
- System.arraycopy(times, 0, timestamps, 0, rowSize);
+ if (rowSize == 0) {
+ timestamps = new long[0];
+ } else if (times != null && times.length >= rowSize) {
+ timestamps = Arrays.copyOf(times, rowSize);
} else {
LOGGER.warn(
"Times array is null or too small. times.length={}, rowSize={},
deviceId={}",
@@ -788,13 +802,15 @@ public class InsertTabletStatement extends
InsertBaseStatement implements ISchem
}
// Get values - convert Statement columns to Tablet format, only for
valid columns
- // All arrays are truncated/copied to rowSize length
+ // All arrays are copied to rowSize length
final Object[] statementColumns = this.getColumns();
final Object[] tabletValues = new Object[schemaSize];
if (statementColumns != null && statementColumns.length > 0) {
- for (int i = 0; i < validColumnIndices.size(); i++) {
- final int originalIndex = validColumnIndices.get(i);
- if (statementColumns[originalIndex] != null &&
dataTypes[originalIndex] != null) {
+ for (int i = 0; i < schemaSize; i++) {
+ final int originalIndex = validColumnIndices[i];
+ if (originalIndex < statementColumns.length
+ && statementColumns[originalIndex] != null
+ && dataTypes[originalIndex] != null) {
tabletValues[i] =
convertColumnToTablet(
statementColumns[originalIndex], dataTypes[originalIndex],
rowSize);
@@ -806,22 +822,25 @@ public class InsertTabletStatement extends
InsertBaseStatement implements ISchem
// Get bitMaps - copy and truncate to rowSize, only for valid columns
final BitMap[] originalBitMaps = this.getBitMaps();
- final BitMap[] bitMaps;
+ BitMap[] bitMaps = null;
if (originalBitMaps != null && originalBitMaps.length > 0) {
- bitMaps = new BitMap[schemaSize];
- for (int i = 0; i < validColumnIndices.size(); i++) {
- final int originalIndex = validColumnIndices.get(i);
- if (originalBitMaps[originalIndex] != null) {
- // Create a new BitMap truncated to rowSize
- final byte[] truncatedBytes =
- originalBitMaps[originalIndex].getTruncatedByteArray(rowSize);
- bitMaps[i] = new BitMap(rowSize, truncatedBytes);
+ final BitMap[] copiedBitMaps = new BitMap[schemaSize];
+ boolean hasMarkedBitMap = false;
+ for (int i = 0; i < schemaSize; i++) {
+ final int originalIndex = validColumnIndices[i];
+ if (originalIndex < originalBitMaps.length &&
originalBitMaps[originalIndex] != null) {
+ final BitMap originalBitMap = originalBitMaps[originalIndex];
+ if (!originalBitMap.isAllUnmarked(Math.min(rowSize,
originalBitMap.getSize()))) {
+ copiedBitMaps[i] = new BitMap(rowSize,
originalBitMap.getTruncatedByteArray(rowSize));
+ hasMarkedBitMap = true;
+ }
} else {
- bitMaps[i] = null;
+ copiedBitMaps[i] = null;
}
}
- } else {
- bitMaps = null;
+ if (hasMarkedBitMap) {
+ bitMaps = copiedBitMaps;
+ }
}
// Create Tablet using the full constructor
@@ -834,7 +853,7 @@ public class InsertTabletStatement extends
InsertBaseStatement implements ISchem
tabletColumnTypes,
timestamps,
tabletValues,
- BitMapUtils.compactBitMaps(bitMaps, rowSize),
+ bitMaps,
rowSize);
} catch (final Exception e) {
throw new MetadataException(
@@ -845,13 +864,13 @@ public class InsertTabletStatement extends
InsertBaseStatement implements ISchem
/**
* Convert a single column value from Statement format to Tablet format.
Statement uses primitive
* arrays (e.g., int[], long[], float[]), while Tablet may need different
format. All arrays are
- * copied and truncated to rowSize length to ensure immutability - even if
the original array is
- * modified, the converted array remains unchanged.
+ * copied to rowSize length to ensure immutability - even if the original
array is modified, the
+ * converted array remains unchanged.
*
* @param columnValue column value from Statement (primitive array)
* @param dataType data type of the column
- * @param rowSize number of rows to copy (truncate to this length)
- * @return column value in Tablet format (copied and truncated array)
+ * @param rowSize number of rows to copy
+ * @return column value in Tablet format (copied to rowSize)
*/
private Object convertColumnToTablet(
final Object columnValue, final TSDataType dataType, final int rowSize) {
@@ -862,39 +881,34 @@ public class InsertTabletStatement extends
InsertBaseStatement implements ISchem
if (TSDataType.DATE.equals(dataType)) {
final int[] values = (int[]) columnValue;
- // Copy and truncate to rowSize
- final int[] copiedValues = Arrays.copyOf(values, Math.min(values.length,
rowSize));
final LocalDate[] localDateValue = new LocalDate[rowSize];
- for (int i = 0; i < copiedValues.length; i++) {
- localDateValue[i] = DateUtils.parseIntToLocalDate(copiedValues[i]);
- }
- // Fill remaining with null if needed
- for (int i = copiedValues.length; i < rowSize; i++) {
- localDateValue[i] = null;
+ final int size = Math.min(values.length, rowSize);
+ for (int i = 0; i < size; i++) {
+ localDateValue[i] = DateUtils.parseIntToLocalDate(values[i]);
}
return localDateValue;
}
- // For primitive arrays, copy and truncate to rowSize
+ // For primitive arrays, copy to rowSize
if (columnValue instanceof boolean[]) {
final boolean[] original = (boolean[]) columnValue;
- return Arrays.copyOf(original, Math.min(original.length, rowSize));
+ return Arrays.copyOf(original, rowSize);
} else if (columnValue instanceof int[]) {
final int[] original = (int[]) columnValue;
- return Arrays.copyOf(original, Math.min(original.length, rowSize));
+ return Arrays.copyOf(original, rowSize);
} else if (columnValue instanceof long[]) {
final long[] original = (long[]) columnValue;
- return Arrays.copyOf(original, Math.min(original.length, rowSize));
+ return Arrays.copyOf(original, rowSize);
} else if (columnValue instanceof float[]) {
final float[] original = (float[]) columnValue;
- return Arrays.copyOf(original, Math.min(original.length, rowSize));
+ return Arrays.copyOf(original, rowSize);
} else if (columnValue instanceof double[]) {
final double[] original = (double[]) columnValue;
- return Arrays.copyOf(original, Math.min(original.length, rowSize));
+ return Arrays.copyOf(original, rowSize);
} else if (columnValue instanceof Binary[]) {
- // For Binary arrays, create a new array and copy references, truncate
to rowSize
+ // For Binary arrays, create a new array and copy references to rowSize
final Binary[] original = (Binary[]) columnValue;
- return Arrays.copyOf(original, Math.min(original.length, rowSize));
+ return Arrays.copyOf(original, rowSize);
}
// For other types, return as-is (should not happen for standard types)
