This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch fix_tablet_column_type_handle in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit ee6309e9f0f9f1836091c2a434bbc240f84f2723 Author: Tian Jiang <[email protected]> AuthorDate: Fri Sep 20 18:13:06 2024 +0800 Fix Column Type handle in Tablet --- .../java/org/apache/tsfile/write/TsFileWriter.java | 7 +------ .../write/chunk/AlignedChunkGroupWriterImpl.java | 18 ++++++++-------- .../tsfile/write/chunk/IChunkGroupWriter.java | 3 --- .../chunk/NonAlignedChunkGroupWriterImpl.java | 17 ++++++++------- .../org/apache/tsfile/write/record/Tablet.java | 24 +++++++++------------- 5 files changed, 28 insertions(+), 41 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java index b1f60735..1733b005 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java @@ -704,12 +704,7 @@ public class TsFileWriter implements AutoCloseable { // get corresponding ChunkGroupWriter and write this Tablet recordCount += tryToInitialGroupWriter(pair.left, isTableWriteAligned) - .write( - tablet, - startIndex, - pair.right, - tablet.getIdColumnRange(), - tablet.getSchemas().size()); + .write(tablet, startIndex, pair.right); startIndex = pair.right; } return checkMemorySizeAndMayFlushChunks(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index 163e583b..41fa5577 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -33,6 +33,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.record.Tablet.ColumnType; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.writer.TsFileIOWriter; @@ -183,17 +184,11 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter { @Override public int write(Tablet tablet) throws IOException, WriteProcessException { - return write(tablet, 0, tablet.rowSize, 0, tablet.getSchemas().size()); - } - - public int write(Tablet tablet, int startRowIndex, int endRowIndex) - throws IOException, WriteProcessException { - return write(tablet, startRowIndex, endRowIndex, 0, tablet.getSchemas().size()); + return write(tablet, 0, tablet.rowSize); } @Override - public int write( - Tablet tablet, int startRowIndex, int endRowIndex, int startColIndex, int endColIndex) + public int write(Tablet tablet, int startRowIndex, int endRowIndex) throws WriteProcessException, IOException { int pointCount = 0; List<IMeasurementSchema> measurementSchemas = tablet.getSchemas(); @@ -213,7 +208,12 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter { for (int row = startRowIndex; row < endRowIndex; row++) { long time = tablet.timestamps[row]; checkIsHistoryData(time); - for (int columnIndex = startColIndex; columnIndex < endColIndex; columnIndex++) { + for (int columnIndex = 0; columnIndex < tablet.getSchemas().size(); columnIndex++) { + if (tablet.getColumnTypes() != null + && tablet.getColumnTypes().get(columnIndex) != ColumnType.MEASUREMENT) { + continue; + } + boolean isNull = tablet.bitMaps != null && tablet.bitMaps[columnIndex] != null diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/IChunkGroupWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/IChunkGroupWriter.java index bc7bc12c..de42c558 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/IChunkGroupWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/IChunkGroupWriter.java @@ -55,9 +55,6 @@ public interface IChunkGroupWriter { int write(Tablet table, int startRowIndex, int endRowIndex) throws WriteProcessException, IOException; - int write(Tablet table, int startRowIndex, int endRowIndex, int startColIndex, int endColIndex) - throws WriteProcessException, IOException; - /** * flushing method for serializing to local file system or HDFS. Implemented by * ChunkWriterImpl.writeToFileWriter(). diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java index 223fffa1..7bbacfe8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java @@ -28,6 +28,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.record.Tablet.ColumnType; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.writer.TsFileIOWriter; @@ -101,21 +102,19 @@ public class NonAlignedChunkGroupWriterImpl implements IChunkGroupWriter { @Override public int write(Tablet tablet) throws IOException, WriteProcessException { - return write(tablet, 0, tablet.rowSize, 0, tablet.getSchemas().size()); - } - - public int write(Tablet tablet, int startRowIndex, int endRowIndex) - throws IOException, WriteProcessException { - return write(tablet, startRowIndex, endRowIndex, 0, tablet.getSchemas().size()); + return write(tablet, 0, tablet.rowSize); } @Override - public int write( - Tablet tablet, int startRowIndex, int endRowIndex, int startColIndex, int endColIndex) + public int write(Tablet tablet, int startRowIndex, int endRowIndex) throws WriteProcessException, IOException { int maxPointCount = 0, pointCount; List<IMeasurementSchema> timeseries = tablet.getSchemas(); - for (int column = startColIndex; column < endColIndex; column++) { + for (int column = 0; column < tablet.getSchemas().size(); column++) { + if (tablet.getColumnTypes() != null + && tablet.getColumnTypes().get(column) != ColumnType.MEASUREMENT) { + continue; + } String measurementId = timeseries.get(column).getMeasurementId(); TSDataType tsDataType = timeseries.get(column).getType(); pointCount = 0; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 253d99b6..c826499d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -71,8 +71,8 @@ public class Tablet { */ private List<ColumnType> columnTypes; - /** Columns in [0, idColumnRange) are all ID columns. */ - private int idColumnRange; + /** Columns in the list are all ID columns. */ + private List<Integer> idColumnIndexes = new ArrayList<>(); /** MeasurementId->indexOf({@link MeasurementSchema}) */ private final Map<String, Integer> measurementIndex; @@ -962,27 +962,23 @@ public class Tablet { * @return the IDeviceID of the i-th row. */ public IDeviceID getDeviceID(int i) { - String[] idArray = new String[idColumnRange + 1]; + String[] idArray = new String[idColumnIndexes.size() + 1]; idArray[0] = insertTargetName; - for (int j = 0; j < idColumnRange; j++) { - final Object value = getValue(i, j); + for (int j = 0; j < idColumnIndexes.size(); j++) { + final Object value = getValue(i, idColumnIndexes.get(j)); idArray[j + 1] = value != null ? value.toString() : null; } return new StringArrayDeviceID(idArray); } - public int getIdColumnRange() { - return idColumnRange; - } - public void setColumnTypes(List<ColumnType> columnTypes) { this.columnTypes = columnTypes; - idColumnRange = 0; - for (ColumnType columnType : columnTypes) { - if (columnType.equals(ColumnType.MEASUREMENT)) { - break; + idColumnIndexes.clear(); + for (int i = 0; i < columnTypes.size(); i++) { + ColumnType columnType = columnTypes.get(i); + if (columnType.equals(ColumnType.ID)) { + idColumnIndexes.add(i); } - idColumnRange++; } }
