This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new f1466e6d Convert column name and table name to lower case (#322)
f1466e6d is described below
commit f1466e6d131eefb8653ba5efd6e05ba842761050
Author: shuwenwei <[email protected]>
AuthorDate: Fri Dec 6 10:06:56 2024 +0800
Convert column name and table name to lower case (#322)
* Only performs the lower case conversion in table model
* modify DeviceTableModelWriter
* fix ut
* modify query
* modify query
* remove null check
---
.../tsfile/v4/WriteTabletWithITsFileWriter.java | 2 +-
.../apache/tsfile/file/metadata/TableSchema.java | 60 +++++++++++++-----
.../tsfile/read/v4/DeviceTableModelReader.java | 13 ++--
.../java/org/apache/tsfile/write/TsFileWriter.java | 15 +++--
.../write/chunk/AlignedChunkGroupWriterImpl.java | 46 +++++++++++---
.../write/chunk/TableChunkGroupWriterImpl.java | 36 +++++++++++
.../org/apache/tsfile/write/record/Tablet.java | 35 +++++++----
.../write/v4/AbstractTableModelTsFileWriter.java | 9 ++-
.../tsfile/write/v4/DeviceTableModelWriter.java | 2 +-
.../apache/tsfile/read/query/ResultSetTest.java | 20 +++---
.../apache/tsfile/tableview/TableSchemaTest.java | 1 +
.../apache/tsfile/write/TsFileWriteApiTest.java | 71 ++++++++++++++++++++++
12 files changed, 250 insertions(+), 60 deletions(-)
diff --git
a/java/examples/src/main/java/org/apache/tsfile/v4/WriteTabletWithITsFileWriter.java
b/java/examples/src/main/java/org/apache/tsfile/v4/WriteTabletWithITsFileWriter.java
index d7dca831..5da781bd 100644
---
a/java/examples/src/main/java/org/apache/tsfile/v4/WriteTabletWithITsFileWriter.java
+++
b/java/examples/src/main/java/org/apache/tsfile/v4/WriteTabletWithITsFileWriter.java
@@ -106,7 +106,7 @@ public class WriteTabletWithITsFileWriter {
tablet.addValue(rowIndex, 0, "id1_field_2");
// id2 column
- tablet.addValue(rowIndex, 1, "id1_field_2");
+ tablet.addValue(rowIndex, 1, "id2_field_2");
// s1 column
tablet.addValue(rowIndex, 2, 1);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
index 21d44425..fcd3b434 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
@@ -37,6 +37,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.stream.Collectors;
public class TableSchema {
@@ -53,18 +54,45 @@ public class TableSchema {
private Map<String, Integer> idColumnOrder;
public TableSchema(String tableName) {
- this.tableName = tableName;
+ this.tableName = tableName.toLowerCase();
this.measurementSchemas = new ArrayList<>();
this.columnCategories = new ArrayList<>();
this.updatable = true;
}
+ // for deserialize
+ public TableSchema(
+ List<IMeasurementSchema> columnSchemas, List<ColumnCategory>
columnCategories) {
+ this.measurementSchemas =
+ columnSchemas.stream()
+ .map(
+ measurementSchema ->
+ new MeasurementSchema(
+ measurementSchema.getMeasurementName().toLowerCase(),
+ measurementSchema.getType(),
+ measurementSchema.getEncodingType(),
+ measurementSchema.getCompressor(),
+ measurementSchema.getProps()))
+ .collect(Collectors.toList());
+ this.columnCategories = columnCategories;
+ }
+
public TableSchema(
String tableName,
List<IMeasurementSchema> columnSchemas,
List<ColumnCategory> columnCategories) {
- this.tableName = tableName;
- this.measurementSchemas = columnSchemas;
+ this.tableName = tableName.toLowerCase();
+ this.measurementSchemas =
+ columnSchemas.stream()
+ .map(
+ measurementSchema ->
+ new MeasurementSchema(
+ measurementSchema.getMeasurementName().toLowerCase(),
+ measurementSchema.getType(),
+ measurementSchema.getEncodingType(),
+ measurementSchema.getCompressor(),
+ measurementSchema.getProps()))
+ .collect(Collectors.toList());
this.columnCategories = columnCategories;
}
@@ -73,22 +101,24 @@ public class TableSchema {
List<String> columnNameList,
List<TSDataType> dataTypeList,
List<ColumnCategory> categoryList) {
- this.tableName = tableName;
+ this.tableName = tableName.toLowerCase();
this.measurementSchemas = new ArrayList<>(columnNameList.size());
for (int i = 0; i < columnNameList.size(); i++) {
- measurementSchemas.add(new MeasurementSchema(columnNameList.get(i),
dataTypeList.get(i)));
+ measurementSchemas.add(
+ new MeasurementSchema(columnNameList.get(i).toLowerCase(),
dataTypeList.get(i)));
}
this.columnCategories = categoryList;
}
@TsFileApi
public TableSchema(String tableName, List<ColumnSchema> columnSchemaList) {
- this.tableName = tableName;
+ this.tableName = tableName.toLowerCase();
this.measurementSchemas = new ArrayList<>(columnSchemaList.size());
this.columnCategories = new ArrayList<>(columnSchemaList.size());
for (ColumnSchema columnSchema : columnSchemaList) {
this.measurementSchemas.add(
- new MeasurementSchema(columnSchema.getColumnName(),
columnSchema.getDataType()));
+ new MeasurementSchema(
+ columnSchema.getColumnName().toLowerCase(),
columnSchema.getDataType()));
this.columnCategories.add(columnSchema.getColumnCategory());
}
}
@@ -126,12 +156,13 @@ public class TableSchema {
* @return i if the given column is the i-th column, -1 if the column is not
in the schema
*/
public int findColumnIndex(String columnName) {
+ final String lowerCaseColumnName = columnName.toLowerCase();
return getColumnPosIndex()
.computeIfAbsent(
- columnName,
+ lowerCaseColumnName,
colName -> {
for (int i = 0; i < measurementSchemas.size(); i++) {
- if
(measurementSchemas.get(i).getMeasurementName().equals(columnName)) {
+ if
(measurementSchemas.get(i).getMeasurementName().equals(lowerCaseColumnName)) {
return i;
}
}
@@ -144,13 +175,14 @@ public class TableSchema {
* not an ID column
*/
public int findIdColumnOrder(String columnName) {
+ final String lowerCaseColumnName = columnName.toLowerCase();
return getIdColumnOrder()
.computeIfAbsent(
- columnName,
+ lowerCaseColumnName,
colName -> {
int columnOrder = 0;
for (int i = 0; i < measurementSchemas.size(); i++) {
- if
(measurementSchemas.get(i).getMeasurementName().equals(columnName)
+ if
(measurementSchemas.get(i).getMeasurementName().equals(lowerCaseColumnName)
&& columnCategories.get(i) == ColumnCategory.ID) {
return columnOrder;
} else if (columnCategories.get(i) == ColumnCategory.ID) {
@@ -162,7 +194,7 @@ public class TableSchema {
}
public IMeasurementSchema findColumnSchema(String columnName) {
- final int columnIndex = findColumnIndex(columnName);
+ final int columnIndex = findColumnIndex(columnName.toLowerCase());
return columnIndex >= 0 ? measurementSchemas.get(columnIndex) : null;
}
@@ -230,7 +262,7 @@ public class TableSchema {
measurementSchemas.add(measurementSchema);
columnCategories.add(ColumnCategory.values()[buffer.getInt()]);
}
- return new TableSchema(null, measurementSchemas, columnCategories);
+ return new TableSchema(measurementSchemas, columnCategories);
}
public String getTableName() {
@@ -238,7 +270,7 @@ public class TableSchema {
}
public void setTableName(String tableName) {
- this.tableName = tableName;
+ this.tableName = tableName.toLowerCase();
}
@Override
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
index a354df77..50f6ab6e 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
@@ -74,30 +74,33 @@ public class DeviceTableModelReader implements
ITsFileReader {
public Optional<TableSchema> getTableSchemas(String tableName) throws
IOException {
TsFileMetadata tsFileMetadata = fileReader.readFileMetadata();
Map<String, TableSchema> tableSchemaMap =
tsFileMetadata.getTableSchemaMap();
- return Optional.ofNullable(tableSchemaMap.get(tableName));
+ return Optional.ofNullable(tableSchemaMap.get(tableName.toLowerCase()));
}
@TsFileApi
public ResultSet query(String tableName, List<String> columnNames, long
startTime, long endTime)
throws IOException, NoTableException, NoMeasurementException,
ReadProcessException {
+ String lowerCaseTableName = tableName.toLowerCase();
TsFileMetadata tsFileMetadata = fileReader.readFileMetadata();
- TableSchema tableSchema =
tsFileMetadata.getTableSchemaMap().get(tableName);
+ TableSchema tableSchema =
tsFileMetadata.getTableSchemaMap().get(lowerCaseTableName);
if (tableSchema == null) {
throw new NoTableException(tableName);
}
List<TSDataType> dataTypeList = new ArrayList<>(columnNames.size());
+ List<String> lowerCaseColumnNames = new ArrayList<>(columnNames.size());
for (String columnName : columnNames) {
Map<String, Integer> column2IndexMap = tableSchema.buildColumnPosIndex();
- Integer columnIndex = column2IndexMap.get(columnName);
+ Integer columnIndex = column2IndexMap.get(columnName.toLowerCase());
if (columnIndex == null) {
throw new NoMeasurementException(columnName);
}
+ lowerCaseColumnNames.add(columnName.toLowerCase());
dataTypeList.add(tableSchema.getColumnSchemas().get(columnIndex).getType());
}
TsBlockReader tsBlockReader =
queryExecutor.query(
- tableName,
- columnNames,
+ lowerCaseTableName,
+ lowerCaseColumnNames,
new ExpressionTree.TimeBetweenAnd(startTime, endTime),
null,
null);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
index a8aae270..d2f77400 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
@@ -38,6 +38,7 @@ import org.apache.tsfile.utils.WriteUtils;
import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl;
import org.apache.tsfile.write.chunk.IChunkGroupWriter;
import org.apache.tsfile.write.chunk.NonAlignedChunkGroupWriterImpl;
+import org.apache.tsfile.write.chunk.TableChunkGroupWriterImpl;
import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.record.datapoint.DataPoint;
@@ -347,7 +348,7 @@ public class TsFileWriter implements AutoCloseable {
throws WriteProcessException, IOException {
// initial ChunkGroupWriter of this device in the TSRecord
final IDeviceID deviceID = record.deviceId;
- IChunkGroupWriter groupWriter = tryToInitialGroupWriter(deviceID,
isAligned);
+ IChunkGroupWriter groupWriter = tryToInitialGroupWriter(deviceID,
isAligned, false);
// initial all SeriesWriters of measurements in this TSRecord
List<IMeasurementSchema> measurementSchemas;
@@ -411,7 +412,7 @@ public class TsFileWriter implements AutoCloseable {
private void checkIsTimeseriesExist(Tablet tablet, boolean isAligned)
throws WriteProcessException, IOException {
final IDeviceID deviceID =
IDeviceID.Factory.DEFAULT_FACTORY.create(tablet.getDeviceId());
- IChunkGroupWriter groupWriter = tryToInitialGroupWriter(deviceID,
isAligned);
+ IChunkGroupWriter groupWriter = tryToInitialGroupWriter(deviceID,
isAligned, false);
List<IMeasurementSchema> schemas = tablet.getSchemas();
if (getSchema().containsDevice(deviceID)) {
@@ -495,11 +496,15 @@ public class TsFileWriter implements AutoCloseable {
return schemas;
}
- private IChunkGroupWriter tryToInitialGroupWriter(IDeviceID deviceId,
boolean isAligned) {
+ private IChunkGroupWriter tryToInitialGroupWriter(
+ IDeviceID deviceId, boolean isAligned, boolean isTableModel) {
IChunkGroupWriter groupWriter = groupWriters.get(deviceId);
if (groupWriter == null) {
if (isAligned) {
- groupWriter = new AlignedChunkGroupWriterImpl(deviceId, encryptParam);
+ groupWriter =
+ isTableModel
+ ? new TableChunkGroupWriterImpl(deviceId, encryptParam)
+ : new AlignedChunkGroupWriterImpl(deviceId, encryptParam);
if (!isUnseq) { // Sequence File
((AlignedChunkGroupWriterImpl) groupWriter)
.setLastTime(alignedDeviceLastTimeMap.get(deviceId));
@@ -733,7 +738,7 @@ public class TsFileWriter implements AutoCloseable {
for (Pair<IDeviceID, Integer> pair : deviceIdEndIndexPairs) {
// get corresponding ChunkGroupWriter and write this Tablet
recordCount +=
- tryToInitialGroupWriter(pair.left, isTableWriteAligned)
+ tryToInitialGroupWriter(pair.left, isTableWriteAligned, true)
.write(tablet, startIndex, pair.right);
startIndex = pair.right;
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
index e50d8a5c..d6dd6c87 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
@@ -64,6 +64,7 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
private long lastTime = Long.MIN_VALUE;
private boolean isInitLastTime = false;
+ private boolean convertColumnNameToLowerCase = false;
public AlignedChunkGroupWriterImpl(IDeviceID deviceId) {
this.deviceId = deviceId;
@@ -100,17 +101,20 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
public ValueChunkWriter tryToAddSeriesWriterInternal(IMeasurementSchema
measurementSchema)
throws IOException {
- ValueChunkWriter valueChunkWriter =
- valueChunkWriterMap.get(measurementSchema.getMeasurementName());
+ String measurementName =
+ convertColumnNameToLowerCase
+ ? measurementSchema.getMeasurementName().toLowerCase()
+ : measurementSchema.getMeasurementName();
+ ValueChunkWriter valueChunkWriter =
valueChunkWriterMap.get(measurementName);
if (valueChunkWriter == null) {
valueChunkWriter =
new ValueChunkWriter(
- measurementSchema.getMeasurementName(),
+ measurementName,
measurementSchema.getCompressor(),
measurementSchema.getType(),
measurementSchema.getEncodingType(),
measurementSchema.getValueEncoder());
- valueChunkWriterMap.put(measurementSchema.getMeasurementName(),
valueChunkWriter);
+ valueChunkWriterMap.put(measurementName, valueChunkWriter);
tryToAddEmptyPageAndData(valueChunkWriter);
}
return valueChunkWriter;
@@ -119,15 +123,19 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
@Override
public void tryToAddSeriesWriter(List<IMeasurementSchema>
measurementSchemas) throws IOException {
for (IMeasurementSchema schema : measurementSchemas) {
- if (!valueChunkWriterMap.containsKey(schema.getMeasurementName())) {
+ String measurementName =
+ convertColumnNameToLowerCase
+ ? schema.getMeasurementName().toLowerCase()
+ : schema.getMeasurementName();
+ if (!valueChunkWriterMap.containsKey(measurementName)) {
ValueChunkWriter valueChunkWriter =
new ValueChunkWriter(
- schema.getMeasurementName(),
+ measurementName,
schema.getCompressor(),
schema.getType(),
schema.getEncodingType(),
schema.getValueEncoder());
- valueChunkWriterMap.put(schema.getMeasurementName(), valueChunkWriter);
+ valueChunkWriterMap.put(measurementName, valueChunkWriter);
tryToAddEmptyPageAndData(valueChunkWriter);
}
}
@@ -138,7 +146,13 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
checkIsHistoryData(time);
List<ValueChunkWriter> emptyValueChunkWriters = new ArrayList<>();
Set<String> existingMeasurements =
-
data.stream().map(DataPoint::getMeasurementId).collect(Collectors.toSet());
+ data.stream()
+ .map(
+ dataPoint ->
+ convertColumnNameToLowerCase
+ ? dataPoint.getMeasurementId().toLowerCase()
+ : dataPoint.getMeasurementId())
+ .collect(Collectors.toSet());
for (Map.Entry<String, ValueChunkWriter> entry :
valueChunkWriterMap.entrySet()) {
if (!existingMeasurements.contains(entry.getKey())) {
emptyValueChunkWriters.add(entry.getValue());
@@ -146,7 +160,11 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
}
for (DataPoint point : data) {
boolean isNull = point.getValue() == null;
- ValueChunkWriter valueChunkWriter =
valueChunkWriterMap.get(point.getMeasurementId());
+ String measurementId =
+ convertColumnNameToLowerCase
+ ? point.getMeasurementId().toLowerCase()
+ : point.getMeasurementId();
+ ValueChunkWriter valueChunkWriter =
valueChunkWriterMap.get(measurementId);
switch (point.getType()) {
case BOOLEAN:
valueChunkWriter.write(time, (boolean) point.getValue(), isNull);
@@ -201,7 +219,11 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
// TODO: should we allow duplicated measurements in a Tablet?
Set<String> existingMeasurements =
measurementSchemas.stream()
- .map(IMeasurementSchema::getMeasurementName)
+ .map(
+ schema ->
+ convertColumnNameToLowerCase
+ ? schema.getMeasurementName().toLowerCase()
+ : schema.getMeasurementName())
.collect(Collectors.toSet());
for (Map.Entry<String, ValueChunkWriter> entry :
valueChunkWriterMap.entrySet()) {
if (!existingMeasurements.contains(entry.getKey())) {
@@ -413,4 +435,8 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
isInitLastTime = true;
}
}
+
+ public void setConvertColumnNameToLowerCase(boolean
convertColumnNameToLowerCase) {
+ this.convertColumnNameToLowerCase = convertColumnNameToLowerCase;
+ }
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TableChunkGroupWriterImpl.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TableChunkGroupWriterImpl.java
new file mode 100644
index 00000000..3e987b66
--- /dev/null
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TableChunkGroupWriterImpl.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.write.chunk;
+
+import org.apache.tsfile.encrypt.EncryptParameter;
+import org.apache.tsfile.file.metadata.IDeviceID;
+
+public class TableChunkGroupWriterImpl extends AlignedChunkGroupWriterImpl {
+
+ public TableChunkGroupWriterImpl(IDeviceID deviceId) {
+ super(deviceId);
+ setConvertColumnNameToLowerCase(true);
+ }
+
+ public TableChunkGroupWriterImpl(IDeviceID deviceId, EncryptParameter
encryptParam) {
+ super(deviceId, encryptParam);
+ setConvertColumnNameToLowerCase(true);
+ }
+}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
index e39d2d06..515b41bb 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
@@ -279,7 +279,7 @@ public class Tablet {
}
public void addValue(final String measurementId, final int rowIndex, final
Object value) {
- int indexOfSchema = measurementIndex.get(measurementId);
+ int indexOfSchema = getColumnIndexByMeasurement(measurementId);
IMeasurementSchema measurementSchema = schemas.get(indexOfSchema);
addValueOfDataType(measurementSchema.getType(), rowIndex, indexOfSchema,
value);
}
@@ -349,7 +349,7 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, String measurement, int val) {
- Integer columnIndex = measurementIndex.get(measurement);
+ int columnIndex = getColumnIndexByMeasurement(measurement);
addValue(rowIndex, columnIndex, val);
}
@@ -362,7 +362,7 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, String measurement, long val) {
- Integer columnIndex = measurementIndex.get(measurement);
+ int columnIndex = getColumnIndexByMeasurement(measurement);
addValue(rowIndex, columnIndex, val);
}
@@ -375,7 +375,7 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, String measurement, float val) {
- Integer columnIndex = measurementIndex.get(measurement);
+ int columnIndex = getColumnIndexByMeasurement(measurement);
addValue(rowIndex, columnIndex, val);
}
@@ -388,7 +388,7 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, String measurement, double val) {
- Integer columnIndex = measurementIndex.get(measurement);
+ int columnIndex = getColumnIndexByMeasurement(measurement);
addValue(rowIndex, columnIndex, val);
}
@@ -401,7 +401,7 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, String measurement, boolean val) {
- Integer columnIndex = measurementIndex.get(measurement);
+ int columnIndex = getColumnIndexByMeasurement(measurement);
addValue(rowIndex, columnIndex, val);
}
@@ -414,7 +414,7 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, String measurement, String val) {
- Integer columnIndex = measurementIndex.get(measurement);
+ int columnIndex = getColumnIndexByMeasurement(measurement);
addValue(rowIndex, columnIndex, val);
}
@@ -427,7 +427,7 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, String measurement, byte[] val) {
- Integer columnIndex = measurementIndex.get(measurement);
+ int columnIndex = getColumnIndexByMeasurement(measurement);
addValue(rowIndex, columnIndex, val);
}
@@ -440,7 +440,7 @@ public class Tablet {
@TsFileApi
public void addValue(int rowIndex, String measurement, LocalDate val) {
- Integer columnIndex = measurementIndex.get(measurement);
+ int columnIndex = getColumnIndexByMeasurement(measurement);
addValue(rowIndex, columnIndex, val);
}
@@ -451,6 +451,17 @@ public class Tablet {
updateBitMap(rowIndex, columnIndex, false);
}
+ private int getColumnIndexByMeasurement(String measurement) {
+ if (measurement == null) {
+ throw new IllegalArgumentException("measurement should be non null
value");
+ }
+ Integer columnIndex = measurementIndex.get(measurement);
+ if (columnIndex == null) {
+ throw new IllegalArgumentException("No measurement for " + measurement);
+ }
+ return columnIndex;
+ }
+
private void updateBitMap(int rowIndex, int columnIndex, boolean mark) {
if (bitMaps == null) {
initBitMaps();
@@ -1093,7 +1104,7 @@ public class Tablet {
*/
public IDeviceID getDeviceID(int i) {
String[] idArray = new String[idColumnIndexes.size() + 1];
- idArray[0] = insertTargetName;
+ idArray[0] = getTableName();
for (int j = 0; j < idColumnIndexes.size(); j++) {
final Object value = getValue(i, idColumnIndexes.get(j));
idArray[j + 1] = value != null ? value.toString() : null;
@@ -1154,7 +1165,7 @@ public class Tablet {
}
public String getTableName() {
- return insertTargetName;
+ return insertTargetName == null ? null : insertTargetName.toLowerCase();
}
/**
@@ -1163,7 +1174,7 @@ public class Tablet {
* @param tableName set the tableName as the insertTargetName
*/
public void setTableName(String tableName) {
- this.insertTargetName = tableName;
+ this.insertTargetName = tableName.toLowerCase();
}
public List<ColumnCategory> getColumnTypes() {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
index 102fcd69..80069d3e 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
@@ -29,6 +29,7 @@ import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl;
import org.apache.tsfile.write.chunk.IChunkGroupWriter;
import org.apache.tsfile.write.chunk.NonAlignedChunkGroupWriterImpl;
+import org.apache.tsfile.write.chunk.TableChunkGroupWriterImpl;
import org.apache.tsfile.write.schema.Schema;
import org.apache.tsfile.write.writer.TsFileIOWriter;
@@ -139,11 +140,15 @@ abstract class AbstractTableModelTsFileWriter implements
ITsFileWriter {
}
}
- protected IChunkGroupWriter tryToInitialGroupWriter(IDeviceID deviceId,
boolean isAligned) {
+ protected IChunkGroupWriter tryToInitialGroupWriter(
+ IDeviceID deviceId, boolean isAligned, boolean isTableModel) {
IChunkGroupWriter groupWriter = groupWriters.get(deviceId);
if (groupWriter == null) {
if (isAligned) {
- groupWriter = new AlignedChunkGroupWriterImpl(deviceId, encryptParam);
+ groupWriter =
+ isTableModel
+ ? new TableChunkGroupWriterImpl(deviceId, encryptParam)
+ : new AlignedChunkGroupWriterImpl(deviceId, encryptParam);
((AlignedChunkGroupWriterImpl) groupWriter)
.setLastTime(alignedDeviceLastTimeMap.get(deviceId));
} else {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
index 117fdc12..4b517759 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
@@ -66,7 +66,7 @@ public class DeviceTableModelWriter extends
AbstractTableModelTsFileWriter {
for (Pair<IDeviceID, Integer> pair : deviceIdEndIndexPairs) {
// get corresponding ChunkGroupWriter and write this Tablet
recordCount +=
- tryToInitialGroupWriter(pair.left, isTableWriteAligned)
+ tryToInitialGroupWriter(pair.left, isTableWriteAligned, true)
.write(table, startIndex, pair.right);
startIndex = pair.right;
}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
index bf8273bb..6ab9dad1 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
@@ -100,19 +100,19 @@ public class ResultSetTest {
try (DeviceTableModelReader tsFileReader = new
DeviceTableModelReader(tsfile);
ResultSet resultSet =
- tsFileReader.query("t1", Arrays.asList("id1", "id2", "s2", "s1"),
0, 2); ) {
+ tsFileReader.query("T1", Arrays.asList("ID1", "ID2", "S2", "S1"),
0, 2); ) {
// id1 id2 s2 s1
ResultSetMetadata resultSetMetadata = resultSet.getMetadata();
// Time id1 id2 s2 s1
Assert.assertEquals("Time", resultSetMetadata.getColumnName(1));
Assert.assertEquals(TSDataType.INT64,
resultSetMetadata.getColumnType(1));
- Assert.assertEquals("id1", resultSetMetadata.getColumnName(2));
+ Assert.assertEquals("ID1", resultSetMetadata.getColumnName(2));
Assert.assertEquals(TSDataType.STRING,
resultSetMetadata.getColumnType(2));
- Assert.assertEquals("id2", resultSetMetadata.getColumnName(3));
+ Assert.assertEquals("ID2", resultSetMetadata.getColumnName(3));
Assert.assertEquals(TSDataType.STRING,
resultSetMetadata.getColumnType(3));
- Assert.assertEquals("s2", resultSetMetadata.getColumnName(4));
+ Assert.assertEquals("S2", resultSetMetadata.getColumnName(4));
Assert.assertEquals(TSDataType.BOOLEAN,
resultSetMetadata.getColumnType(4));
- Assert.assertEquals("s1", resultSetMetadata.getColumnName(5));
+ Assert.assertEquals("S1", resultSetMetadata.getColumnName(5));
Assert.assertEquals(TSDataType.BOOLEAN,
resultSetMetadata.getColumnType(5));
Assert.assertTrue(resultSet.next());
@@ -177,7 +177,7 @@ public class ResultSetTest {
try (DeviceTableModelReader tsFileReader = new
DeviceTableModelReader(tsfile);
ResultSet resultSet =
- tsFileReader.query("t1", Arrays.asList("id1", "id2", "s2", "s1"),
0, 2); ) {
+ tsFileReader.query("T1", Arrays.asList("id1", "id2", "S2", "S1"),
0, 2); ) {
// id1 id2 s2 s1
ResultSetMetadata resultSetMetadata = resultSet.getMetadata();
// Time id1 id2 s2 s1
@@ -187,9 +187,9 @@ public class ResultSetTest {
Assert.assertEquals(TSDataType.STRING,
resultSetMetadata.getColumnType(2));
Assert.assertEquals("id2", resultSetMetadata.getColumnName(3));
Assert.assertEquals(TSDataType.STRING,
resultSetMetadata.getColumnType(3));
- Assert.assertEquals("s2", resultSetMetadata.getColumnName(4));
+ Assert.assertEquals("S2", resultSetMetadata.getColumnName(4));
Assert.assertEquals(TSDataType.BOOLEAN,
resultSetMetadata.getColumnType(4));
- Assert.assertEquals("s1", resultSetMetadata.getColumnName(5));
+ Assert.assertEquals("S1", resultSetMetadata.getColumnName(5));
Assert.assertEquals(TSDataType.BOOLEAN,
resultSetMetadata.getColumnType(5));
Assert.assertTrue(resultSet.next());
@@ -203,8 +203,8 @@ public class ResultSetTest {
Assert.assertEquals(1, resultSet.getLong(1));
Assert.assertEquals("id_field1", resultSet.getString(2));
Assert.assertEquals("id_field2", resultSet.getString(3));
- Assert.assertTrue(resultSet.isNull("s1"));
- Assert.assertFalse(resultSet.getBoolean("s2"));
+ Assert.assertTrue(resultSet.isNull("S1"));
+ Assert.assertFalse(resultSet.getBoolean("S2"));
}
}
}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableSchemaTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableSchemaTest.java
index 26aa7894..bdd2dcb3 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableSchemaTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableSchemaTest.java
@@ -117,6 +117,7 @@ public class TableSchemaTest {
final TableSchema deserialized = TableSchema.deserialize(buffer, new
DeserializeConfig());
deserialized.setTableName(tableName);
assertEquals(tableSchema, deserialized);
+
assertEquals(measurementSchemaCnt + 2,
deserialized.getColumnSchemas().size());
}
}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
index 788685bd..f4335939 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
@@ -26,7 +26,9 @@ import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.header.ChunkHeader;
import org.apache.tsfile.file.header.PageHeader;
import org.apache.tsfile.file.metadata.ChunkMetadata;
+import org.apache.tsfile.file.metadata.ColumnSchema;
import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.read.TsFileReader;
@@ -58,6 +60,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
public class TsFileWriteApiTest {
private final File f =
FSFactoryProducer.getFSFactory().getFile("TsFileWriteTest.tsfile");
@@ -849,4 +852,72 @@ public class TsFileWriteApiTest {
throw throwable;
}
}
+
+ @Test
+ public void writeTreeTsFileWithUpperCaseColumns() throws IOException,
WriteProcessException {
+ setEnv(100 * 1024 * 1024, 10 * 1024);
+ String d1 = "root.TEST.D1";
+ try (TsFileWriter writer = new TsFileWriter(f)) {
+ writer.registerTimeseries(d1, new MeasurementSchema("MEASUREMENT1",
TSDataType.BOOLEAN));
+ TSRecord record = new TSRecord(d1, 1);
+ record.addPoint("MEASUREMENT1", true);
+ writer.writeRecord(record);
+ }
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getPath())) {
+ Assert.assertTrue(
+
reader.getAllDevices().contains(IDeviceID.Factory.DEFAULT_FACTORY.create(d1)));
+
Assert.assertTrue(reader.getAllMeasurements().containsKey("MEASUREMENT1"));
+ }
+
+ Tablet tablet =
+ new Tablet(d1, Arrays.asList(new MeasurementSchema("MEASUREMENT1",
TSDataType.BOOLEAN)));
+ tablet.addTimestamp(0, 0);
+ tablet.addValue("MEASUREMENT1", 0, true);
+ try (TsFileWriter writer = new TsFileWriter(f)) {
+ writer.registerTimeseries(d1, new MeasurementSchema("MEASUREMENT1",
TSDataType.BOOLEAN));
+ writer.writeTree(tablet);
+ }
+
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getPath())) {
+ Assert.assertTrue(
+
reader.getAllDevices().contains(IDeviceID.Factory.DEFAULT_FACTORY.create(d1)));
+
Assert.assertTrue(reader.getAllMeasurements().containsKey("MEASUREMENT1"));
+ }
+ }
+
+ @Test
+ public void writeTableTsFileWithUpperCaseColumns() throws IOException,
WriteProcessException {
+ setEnv(100 * 1024 * 1024, 10 * 1024);
+ Tablet tablet =
+ new Tablet(
+ "TABLE1",
+ Arrays.asList("IdColumn", "MeasurementColumn"),
+ Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN),
+ Arrays.asList(Tablet.ColumnCategory.ID,
Tablet.ColumnCategory.MEASUREMENT));
+ tablet.addTimestamp(0, 0);
+ tablet.addValue("IdColumn", 0, "id_field");
+ tablet.addValue("MeasurementColumn", 0, true);
+ TableSchema tableSchema =
+ new TableSchema(
+ "Table1",
+ Arrays.asList(
+ new ColumnSchema("IDCOLUMN", TSDataType.STRING,
Tablet.ColumnCategory.ID),
+ new ColumnSchema(
+ "MeasurementColumn", TSDataType.BOOLEAN,
Tablet.ColumnCategory.MEASUREMENT)));
+ Assert.assertEquals("table1", tableSchema.getTableName());
+ try (TsFileWriter writer = new TsFileWriter(f)) {
+ writer.registerTableSchema(tableSchema);
+ writer.writeTable(tablet);
+ }
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getPath())) {
+ Map<String, TableSchema> tableSchemaMap =
reader.readFileMetadata().getTableSchemaMap();
+ TableSchema tableSchemaInTsFile = tableSchemaMap.get("table1");
+ Assert.assertNotNull(tableSchemaInTsFile);
+ for (IMeasurementSchema columnSchema :
tableSchemaInTsFile.getColumnSchemas()) {
+ Assert.assertEquals(
+ columnSchema.getMeasurementName().toLowerCase(),
columnSchema.getMeasurementName());
+ }
+
Assert.assertTrue(reader.getAllMeasurements().containsKey("measurementcolumn"));
+ }
+ }
}
