This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch tsFile_v4
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/tsFile_v4 by this push:
new ed0630b6 add more tests
ed0630b6 is described below
commit ed0630b61a72c6874cff1371740d2f496a3a5a1f
Author: jt2594838 <[email protected]>
AuthorDate: Fri Apr 12 16:26:45 2024 +0800
add more tests
---
.../apache/tsfile/file/metadata/ChunkMetadata.java | 2 +
.../read/controller/MetadataQuerierByFileImpl.java | 17 +-
.../apache/tsfile/write/TableViewWriteTest.java | 182 +++++++++++++++++++++
3 files changed, 188 insertions(+), 13 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java
index 62d6eede..6a766c8a 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java
@@ -108,6 +108,8 @@ public class ChunkMetadata implements IChunkMetadata {
this.tsDataType = tsDataType;
this.offsetOfChunkHeader = fileOffset;
this.statistics = statistics;
+ this.encoding = encoding;
+ this.compressionType = compressionType;
}
// won't clone deleteIntervalList & modified
diff --git
a/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
b/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
index c38e8505..af3039ba 100644
---
a/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
+++
b/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
@@ -56,8 +56,7 @@ public class MetadataQuerierByFileImpl implements
IMetadataQuerier {
private TsFileMetadata fileMetaData;
- // TimeseriesPath -> List<IChunkMetadata>
- private LRUCache<Path, List<IChunkMetadata>> chunkMetaDataCache;
+ // (deviceId, measurementId) -> List<IChunkMetadata>
private LRUCache<Pair<IDeviceID, String>, List<IChunkMetadata>>
deviceIdChunkMetadataCache;
private TsFileSequenceReader tsFileReader;
@@ -66,13 +65,6 @@ public class MetadataQuerierByFileImpl implements
IMetadataQuerier {
public MetadataQuerierByFileImpl(TsFileSequenceReader tsFileReader) throws
IOException {
this.tsFileReader = tsFileReader;
this.fileMetaData = tsFileReader.readFileMetadata();
- chunkMetaDataCache =
- new LRUCache<Path, List<IChunkMetadata>>(CACHED_ENTRY_NUMBER) {
- @Override
- public List<IChunkMetadata> loadObjectByKey(Path key) throws
IOException {
- return loadChunkMetadata(key);
- }
- };
deviceIdChunkMetadataCache =
new LRUCache<Pair<IDeviceID, String>,
List<IChunkMetadata>>(CACHED_ENTRY_NUMBER) {
@Override
@@ -85,7 +77,7 @@ public class MetadataQuerierByFileImpl implements
IMetadataQuerier {
@Override
public List<IChunkMetadata> getChunkMetaDataList(Path timeseriesPath) throws
IOException {
- return new ArrayList<>(chunkMetaDataCache.get(timeseriesPath));
+ return new ArrayList<>(deviceIdChunkMetadataCache.get(new
Pair<>(timeseriesPath.getIDeviceID(), timeseriesPath.getMeasurement())));
}
public List<List<IChunkMetadata>> getChunkMetadataLists(
@@ -173,8 +165,7 @@ public class MetadataQuerierByFileImpl implements
IMetadataQuerier {
} else {
measurementId = ((TimeseriesMetadata)
timeseriesMetadata).getMeasurementId();
}
- this.chunkMetaDataCache.put(
- new Path(selectedDevice, measurementId, true), chunkMetadataList);
+ this.deviceIdChunkMetadataCache.put(new Pair<>(selectedDevice,
measurementId), chunkMetadataList);
count += chunkMetadataList.size();
if (count == CACHED_ENTRY_NUMBER) {
enough = true;
@@ -298,7 +289,7 @@ public class MetadataQuerierByFileImpl implements
IMetadataQuerier {
@Override
public void clear() {
- chunkMetaDataCache.clear();
+ deviceIdChunkMetadataCache.clear();
}
@Override
diff --git
a/tsfile/src/test/java/org/apache/tsfile/write/TableViewWriteTest.java
b/tsfile/src/test/java/org/apache/tsfile/write/TableViewWriteTest.java
index a9133af1..d187e8a2 100644
--- a/tsfile/src/test/java/org/apache/tsfile/write/TableViewWriteTest.java
+++ b/tsfile/src/test/java/org/apache/tsfile/write/TableViewWriteTest.java
@@ -22,19 +22,29 @@ package org.apache.tsfile.write;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.read.ReadProcessException;
import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.IDeviceID.Factory;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.read.common.block.TsBlock;
import org.apache.tsfile.read.controller.CachedChunkLoaderImpl;
import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl;
+import org.apache.tsfile.read.expression.QueryExpression;
+import org.apache.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.tsfile.read.query.executor.QueryExecutor;
import org.apache.tsfile.read.query.executor.TableQueryExecutor;
import
org.apache.tsfile.read.query.executor.TableQueryExecutor.TableQueryOrdering;
+import org.apache.tsfile.read.query.executor.TsFileExecutor;
import org.apache.tsfile.read.reader.block.TsBlockReader;
import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.record.Tablet.ColumnType;
+import org.apache.tsfile.write.record.datapoint.LongDataPoint;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.apache.commons.io.FileUtils;
@@ -101,6 +111,178 @@ public class TableViewWriteTest {
assertEquals(100, cnt);
}
+ @Test
+ public void testWriteMultipleTables() throws Exception {
+ final File testFile = new File(testDir, "testFile");
+ TsFileWriter writer = new TsFileWriter(testFile);
+ List<TableSchema> tableSchemas = new ArrayList<>();
+
+ int tableNum = 10;
+ for (int i = 0; i < tableNum; i++) {
+ final TableSchema tableSchema = genTableSchema(i);
+ tableSchemas.add(tableSchema);
+ writer.registerTableSchema(tableSchema);
+ }
+
+ for (int i = 0; i < tableNum; i++) {
+ writer.writeTable(genTablet(tableSchemas.get(i), 0, 100));
+ }
+ writer.close();
+
+ TsFileSequenceReader sequenceReader = new
TsFileSequenceReader(testFile.getAbsolutePath());
+ TableQueryExecutor tableQueryExecutor =
+ new TableQueryExecutor(
+ new MetadataQuerierByFileImpl(sequenceReader),
+ new CachedChunkLoaderImpl(sequenceReader),
+ TableQueryOrdering.DEVICE);
+
+ final List<String> columns =
+ testTableSchema.getColumnSchemas().stream()
+ .map(MeasurementSchema::getMeasurementId)
+ .collect(Collectors.toList());
+
+ for (int i = 0; i < tableNum; i++) {
+ int cnt;
+ try (TsBlockReader reader =
tableQueryExecutor.query(tableSchemas.get(i).getTableName(),
+ columns, null, null, null)) {
+ assertTrue(reader.hasNext());
+ cnt = 0;
+ while (reader.hasNext()) {
+ final TsBlock result = reader.next();
+ cnt += result.getPositionCount();
+ }
+ }
+ assertEquals(100, cnt);
+ }
+ }
+
+ @Test
+ public void testHybridWrite() throws Exception {
+ final File testFile = new File(testDir, "testFile");
+ TsFileWriter writer = new TsFileWriter(testFile);
+ // table-view registration
+ writer.registerTableSchema(testTableSchema);
+ // tree-view registration
+ final IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.a.b.c.d1");
+ List<MeasurementSchema> treeSchemas = new ArrayList<>();
+ for (int i = 0; i < measurementSchemaNum; i++) {
+ final MeasurementSchema measurementSchema = new MeasurementSchema("s" +
i, TSDataType.INT64,
+ TSEncoding.PLAIN,
+ CompressionType.UNCOMPRESSED);
+ treeSchemas.add(measurementSchema);
+ writer.registerTimeseries(deviceID, measurementSchema);
+ }
+
+ // table-view write
+ final Tablet tablet = genTablet(testTableSchema, 0, 100);
+ writer.writeTable(tablet);
+ // tree-view write
+ for (int i = 0; i < 50; i++) {
+ final TSRecord tsRecord = new TSRecord(i, deviceID);
+ for (int j = 0; j < measurementSchemaNum; j++) {
+ tsRecord.addTuple(new LongDataPoint("s" + j, i));
+ }
+ writer.write(tsRecord);
+ }
+ writer.close();
+
+ // table-view read table-view
+ int cnt;
+ try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(
+ testFile.getAbsolutePath())) {
+ TableQueryExecutor tableQueryExecutor =
+ new TableQueryExecutor(
+ new MetadataQuerierByFileImpl(sequenceReader),
+ new CachedChunkLoaderImpl(sequenceReader),
+ TableQueryOrdering.DEVICE);
+
+ List<String> columns =
+ testTableSchema.getColumnSchemas().stream()
+ .map(MeasurementSchema::getMeasurementId)
+ .collect(Collectors.toList());
+ TsBlockReader reader =
+ tableQueryExecutor.query(testTableSchema.getTableName(), columns,
null, null, null);
+ assertTrue(reader.hasNext());
+ cnt = 0;
+ while (reader.hasNext()) {
+ final TsBlock result = reader.next();
+ cnt += result.getPositionCount();
+ }
+ assertEquals(100, cnt);
+ }
+
+ // tree-view read tree-view
+ try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(
+ testFile.getAbsolutePath())) {
+ QueryExecutor queryExecutor = new TsFileExecutor(
+ new MetadataQuerierByFileImpl(sequenceReader),
+ new CachedChunkLoaderImpl(sequenceReader));
+
+ List<Path> selectedSeries = new ArrayList<>();
+ for (int i = 0; i < measurementSchemaNum; i++) {
+ selectedSeries.add(new Path(deviceID, "s" + i, false));
+ }
+ final QueryExpression queryExpression =
QueryExpression.create(selectedSeries, null);
+ final QueryDataSet queryDataSet = queryExecutor.execute(queryExpression);
+ cnt = 0;
+ while (queryDataSet.hasNext()) {
+ queryDataSet.next();
+ cnt++;
+ }
+ assertEquals(50, cnt);
+ }
+
+ // table-view read tree-view
+ try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(
+ testFile.getAbsolutePath())) {
+ TableQueryExecutor tableQueryExecutor =
+ new TableQueryExecutor(
+ new MetadataQuerierByFileImpl(sequenceReader),
+ new CachedChunkLoaderImpl(sequenceReader),
+ TableQueryOrdering.DEVICE);
+
+ List<String> columns =
+ treeSchemas.stream()
+ .map(MeasurementSchema::getMeasurementId)
+ .collect(Collectors.toList());
+ TsBlockReader reader =
+ tableQueryExecutor.query(deviceID.getTableName(), columns, null,
null, null);
+ assertTrue(reader.hasNext());
+ cnt = 0;
+ while (reader.hasNext()) {
+ final TsBlock result = reader.next();
+ cnt += result.getPositionCount();
+ }
+ assertEquals(50, cnt);
+ }
+
+ // tree-view read table-view
+ try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(
+ testFile.getAbsolutePath())) {
+ QueryExecutor queryExecutor = new TsFileExecutor(
+ new MetadataQuerierByFileImpl(sequenceReader),
+ new CachedChunkLoaderImpl(sequenceReader));
+
+ List<Path> selectedSeries = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ final IDeviceID tabletDeviceID = tablet.getDeviceID(i);
+ for (int j = 0; j < measurementSchemaNum; j++) {
+ selectedSeries.add(new Path(tabletDeviceID, "s" + j, false));
+ }
+ }
+
+ final QueryExpression queryExpression =
QueryExpression.create(selectedSeries, null);
+ final QueryDataSet queryDataSet = queryExecutor.execute(queryExpression);
+ cnt = 0;
+ List<RowRecord> rowRecords = new ArrayList<>();
+ while (queryDataSet.hasNext()) {
+ rowRecords.add(queryDataSet.next());
+ cnt++;
+ }
+ assertEquals(100, cnt);
+ }
+ }
+
private Tablet genTablet(TableSchema tableSchema, int offset, int num) {
Tablet tablet = new Tablet(tableSchema.getTableName(),
tableSchema.getColumnSchemas(),
tableSchema.getColumnTypes());
