This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch no_assume_id_column_first in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 391b55467e8ef5fe40a2f58763394e4e3a953554 Author: Tian Jiang <[email protected]> AuthorDate: Fri Oct 18 14:21:02 2024 +0800 Remove the assumption that id columns should be the first. --- .../apache/tsfile/file/metadata/TableSchema.java | 37 ++++++++++++++- .../reader/block/SingleDeviceTsBlockReader.java | 2 +- .../org/apache/tsfile/tableview/TableViewTest.java | 53 ++++++++++++++++++++-- 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java index 474c4a03..805c4834 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Objects; public class TableSchema { + // the tableName is not serialized since the TableSchema is always stored in a Map, from whose // key the tableName can be known protected String tableName; @@ -45,8 +46,10 @@ public class TableSchema { protected List<ColumnType> columnTypes; protected boolean updatable = false; - // columnName -> pos in columnSchemas; + // columnName -> pos in columnSchemas private Map<String, Integer> columnPosIndex; + // columnName -> pos in all id columns + private Map<String, Integer> idColumnOrder; public TableSchema(String tableName) { this.tableName = tableName; @@ -69,6 +72,16 @@ public class TableSchema { return columnPosIndex; } + public Map<String, Integer> getIdColumnOrder() { + if (idColumnOrder == null) { + idColumnOrder = new HashMap<>(); + } + return idColumnOrder; + } + + /** + * @return i if the given column is the i-th column, -1 if the column is not in the schema + */ public int findColumnIndex(String columnName) { return getColumnPosIndex() .computeIfAbsent( @@ -83,6 +96,28 @@ public class TableSchema { }); } + /** + * @return i if the given column is the i-th ID column, -1 if the column is not in the schema or + * not an ID column + */ + public int findIdColumnOrder(String columnName) { + return getIdColumnOrder() + .computeIfAbsent( + columnName, + colName -> { + int columnOrder = 0; + for (int i = 0; i < columnSchemas.size(); i++) { + if (columnSchemas.get(i).getMeasurementId().equals(columnName) + && columnTypes.get(i) == ColumnType.ID) { + return columnOrder; + } else if (columnTypes.get(i) == ColumnType.ID) { + columnOrder++; + } + } + return -1; + }); + } + public IMeasurementSchema findColumnSchema(String columnName) { final int columnIndex = findColumnIndex(columnName); return columnIndex >= 0 ? columnSchemas.get(columnIndex) : null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java index 473c3127..2051e958 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java @@ -92,7 +92,7 @@ public class SingleDeviceTsBlockReader implements TsBlockReader { for (String idColumn : task.getColumnMapping().getIdColumns()) { final List<Integer> columnPosInResult = task.getColumnMapping().getColumnPos(idColumn); // the first segment in DeviceId is the table name - final int columnPosInId = task.getTableSchema().findColumnIndex(idColumn) + 1; + final int columnPosInId = task.getTableSchema().findIdColumnOrder(idColumn) + 1; idColumnContextMap.put(idColumn, new IdColumnContext(columnPosInResult, columnPosInId)); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java index 95fcc66b..3332a4af 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java @@ -93,14 +93,26 @@ public class TableViewTest { assertEquals(tablet, deserialized); } + @Test + public void testWriterWithIDOrderUnfixed() + throws IOException, WriteProcessException, ReadProcessException { + TableSchema tableSchema = genMixedTableSchema(0); + testWrite(tableSchema); + } + @Test public void testWriteOneTable() throws IOException, WriteProcessException, ReadProcessException { + testWrite(testTableSchema); + } + + private void testWrite(TableSchema tableSchema) + throws IOException, WriteProcessException, ReadProcessException { final File testFile = new File(testDir, "testFile"); TsFileWriter writer = new TsFileWriter(testFile); writer.setGenerateTableSchema(true); - writer.registerTableSchema(testTableSchema); + writer.registerTableSchema(tableSchema); - writer.writeTable(genTablet(testTableSchema, 0, 100)); + writer.writeTable(genTablet(tableSchema, 0, 100)); writer.close(); TsFileSequenceReader sequenceReader = new TsFileSequenceReader(testFile.getAbsolutePath()); @@ -111,18 +123,18 @@ public class TableViewTest { TableQueryOrdering.DEVICE); final List<String> columns = - testTableSchema.getColumnSchemas().stream() + tableSchema.getColumnSchemas().stream() .map(IMeasurementSchema::getMeasurementId) .collect(Collectors.toList()); final TsBlockReader reader = - tableQueryExecutor.query(testTableSchema.getTableName(), columns, null, null, null); + tableQueryExecutor.query(tableSchema.getTableName(), columns, null, null, null); assertTrue(reader.hasNext()); int cnt = 0; while (reader.hasNext()) { final TsBlock result = reader.next(); for (int i = 0; i < result.getPositionCount(); i++) { String col = result.getColumn(0).getObject(i).toString(); - for (int j = 1; j < testTableSchema.getColumnSchemas().size(); j++) { + for (int j = 1; j < tableSchema.getColumnSchemas().size(); j++) { assertEquals(col, result.getColumn(j).getObject(i).toString()); } } @@ -399,4 +411,35 @@ public class TableViewTest { } return new TableSchema("testTable" + tableNum, measurementSchemas, columnTypes); } + + private TableSchema genMixedTableSchema(int tableNum) { + List<IMeasurementSchema> measurementSchemas = new ArrayList<>(); + List<ColumnType> columnTypes = new ArrayList<>(); + + int idIndex = 0; + int measurementIndex = 0; + + while (idIndex < idSchemaNum || measurementIndex < measurementSchemaNum) { + if (idIndex < idSchemaNum) { + measurementSchemas.add( + new MeasurementSchema( + "id" + idIndex, TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED)); + columnTypes.add(ColumnType.ID); + idIndex++; + } + + if (measurementIndex < measurementSchemaNum) { + measurementSchemas.add( + new MeasurementSchema( + "s" + measurementIndex, + TSDataType.INT64, + TSEncoding.PLAIN, + CompressionType.UNCOMPRESSED)); + columnTypes.add(ColumnType.MEASUREMENT); + measurementIndex++; + } + } + + return new TableSchema("testTable" + tableNum, measurementSchemas, columnTypes); + } }
