This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/iotdb by this push:
new 50e9236f Remove the assumption that id columns should be the first.
50e9236f is described below
commit 50e9236f35dd0b348a1463bacf86715c804a110b
Author: Tian Jiang <[email protected]>
AuthorDate: Fri Oct 18 14:21:02 2024 +0800
Remove the assumption that id columns should be the first.
(cherry picked from commit 391b55467e8ef5fe40a2f58763394e4e3a953554)
---
.../apache/tsfile/file/metadata/TableSchema.java | 37 ++++++++++++++-
.../reader/block/SingleDeviceTsBlockReader.java | 2 +-
.../org/apache/tsfile/tableview/TableViewTest.java | 53 ++++++++++++++++++++--
3 files changed, 85 insertions(+), 7 deletions(-)
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
index 474c4a03..805c4834 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
@@ -38,6 +38,7 @@ import java.util.Map;
import java.util.Objects;
public class TableSchema {
+
// the tableName is not serialized since the TableSchema is always stored in
a Map, from whose
// key the tableName can be known
protected String tableName;
@@ -45,8 +46,10 @@ public class TableSchema {
protected List<ColumnType> columnTypes;
protected boolean updatable = false;
- // columnName -> pos in columnSchemas;
+ // columnName -> pos in columnSchemas
private Map<String, Integer> columnPosIndex;
+ // columnName -> pos in all id columns
+ private Map<String, Integer> idColumnOrder;
public TableSchema(String tableName) {
this.tableName = tableName;
@@ -69,6 +72,16 @@ public class TableSchema {
return columnPosIndex;
}
+ public Map<String, Integer> getIdColumnOrder() {
+ if (idColumnOrder == null) {
+ idColumnOrder = new HashMap<>();
+ }
+ return idColumnOrder;
+ }
+
+ /**
+ * @return i if the given column is the i-th column, -1 if the column is not
in the schema
+ */
public int findColumnIndex(String columnName) {
return getColumnPosIndex()
.computeIfAbsent(
@@ -83,6 +96,28 @@ public class TableSchema {
});
}
+ /**
+ * @return i if the given column is the i-th ID column, -1 if the column is
not in the schema or
+ * not an ID column
+ */
+ public int findIdColumnOrder(String columnName) {
+ return getIdColumnOrder()
+ .computeIfAbsent(
+ columnName,
+ colName -> {
+ int columnOrder = 0;
+ for (int i = 0; i < columnSchemas.size(); i++) {
+ if (columnSchemas.get(i).getMeasurementId().equals(columnName)
+ && columnTypes.get(i) == ColumnType.ID) {
+ return columnOrder;
+ } else if (columnTypes.get(i) == ColumnType.ID) {
+ columnOrder++;
+ }
+ }
+ return -1;
+ });
+ }
+
public IMeasurementSchema findColumnSchema(String columnName) {
final int columnIndex = findColumnIndex(columnName);
return columnIndex >= 0 ? columnSchemas.get(columnIndex) : null;
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
index 473c3127..2051e958 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
@@ -92,7 +92,7 @@ public class SingleDeviceTsBlockReader implements
TsBlockReader {
for (String idColumn : task.getColumnMapping().getIdColumns()) {
final List<Integer> columnPosInResult =
task.getColumnMapping().getColumnPos(idColumn);
// the first segment in DeviceId is the table name
- final int columnPosInId =
task.getTableSchema().findColumnIndex(idColumn) + 1;
+ final int columnPosInId =
task.getTableSchema().findIdColumnOrder(idColumn) + 1;
idColumnContextMap.put(idColumn, new IdColumnContext(columnPosInResult,
columnPosInId));
}
}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java
index 95fcc66b..3332a4af 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java
@@ -93,14 +93,26 @@ public class TableViewTest {
assertEquals(tablet, deserialized);
}
+ @Test
+ public void testWriterWithIDOrderUnfixed()
+ throws IOException, WriteProcessException, ReadProcessException {
+ TableSchema tableSchema = genMixedTableSchema(0);
+ testWrite(tableSchema);
+ }
+
@Test
public void testWriteOneTable() throws IOException, WriteProcessException,
ReadProcessException {
+ testWrite(testTableSchema);
+ }
+
+ private void testWrite(TableSchema tableSchema)
+ throws IOException, WriteProcessException, ReadProcessException {
final File testFile = new File(testDir, "testFile");
TsFileWriter writer = new TsFileWriter(testFile);
writer.setGenerateTableSchema(true);
- writer.registerTableSchema(testTableSchema);
+ writer.registerTableSchema(tableSchema);
- writer.writeTable(genTablet(testTableSchema, 0, 100));
+ writer.writeTable(genTablet(tableSchema, 0, 100));
writer.close();
TsFileSequenceReader sequenceReader = new
TsFileSequenceReader(testFile.getAbsolutePath());
@@ -111,18 +123,18 @@ public class TableViewTest {
TableQueryOrdering.DEVICE);
final List<String> columns =
- testTableSchema.getColumnSchemas().stream()
+ tableSchema.getColumnSchemas().stream()
.map(IMeasurementSchema::getMeasurementId)
.collect(Collectors.toList());
final TsBlockReader reader =
- tableQueryExecutor.query(testTableSchema.getTableName(), columns,
null, null, null);
+ tableQueryExecutor.query(tableSchema.getTableName(), columns, null,
null, null);
assertTrue(reader.hasNext());
int cnt = 0;
while (reader.hasNext()) {
final TsBlock result = reader.next();
for (int i = 0; i < result.getPositionCount(); i++) {
String col = result.getColumn(0).getObject(i).toString();
- for (int j = 1; j < testTableSchema.getColumnSchemas().size(); j++) {
+ for (int j = 1; j < tableSchema.getColumnSchemas().size(); j++) {
assertEquals(col, result.getColumn(j).getObject(i).toString());
}
}
@@ -399,4 +411,35 @@ public class TableViewTest {
}
return new TableSchema("testTable" + tableNum, measurementSchemas,
columnTypes);
}
+
+ private TableSchema genMixedTableSchema(int tableNum) {
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<ColumnType> columnTypes = new ArrayList<>();
+
+ int idIndex = 0;
+ int measurementIndex = 0;
+
+ while (idIndex < idSchemaNum || measurementIndex < measurementSchemaNum) {
+ if (idIndex < idSchemaNum) {
+ measurementSchemas.add(
+ new MeasurementSchema(
+ "id" + idIndex, TSDataType.TEXT, TSEncoding.PLAIN,
CompressionType.UNCOMPRESSED));
+ columnTypes.add(ColumnType.ID);
+ idIndex++;
+ }
+
+ if (measurementIndex < measurementSchemaNum) {
+ measurementSchemas.add(
+ new MeasurementSchema(
+ "s" + measurementIndex,
+ TSDataType.INT64,
+ TSEncoding.PLAIN,
+ CompressionType.UNCOMPRESSED));
+ columnTypes.add(ColumnType.MEASUREMENT);
+ measurementIndex++;
+ }
+ }
+
+ return new TableSchema("testTable" + tableNum, measurementSchemas,
columnTypes);
+ }
}
