This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 138c1aa6 Revert "supplement perf test"
138c1aa6 is described below
commit 138c1aa6b4e7fad34da0b9a203b2f882fcdd7a11
Author: jt2594838 <[email protected]>
AuthorDate: Tue Apr 16 12:27:37 2024 +0800
Revert "supplement perf test"
This reverts commit b1ff701d7e876e9b7e2267c30bbdbde74eb10875.
---
.../read/controller/MetadataQuerierByFileImpl.java | 2 +-
.../java/org/apache/tsfile/write/TsFileWriter.java | 2 +-
.../org/apache/tsfile/write/PerformanceTest.java | 223 ---------------------
.../org/apache/tsfile/write/TsFileWriterTest.java | 16 +-
4 files changed, 11 insertions(+), 232 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
b/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
index 8259b60e..64d62442 100644
---
a/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
+++
b/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
@@ -133,7 +133,7 @@ public class MetadataQuerierByFileImpl implements
IMetadataQuerier {
measurementId = ((TimeseriesMetadata)
timeseriesMetadata).getMeasurementId();
}
this.chunkMetaDataCache.put(
- new Path(selectedDevice, measurementId, false), chunkMetadataList);
+ new Path(selectedDevice, measurementId, true), chunkMetadataList);
count += chunkMetadataList.size();
if (count == CACHED_ENTRY_NUMBER) {
enough = true;
diff --git a/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
b/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
index 2b0ca722..cdbb8b32 100644
--- a/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
@@ -356,7 +356,7 @@ public class TsFileWriter implements AutoCloseable {
IChunkGroupWriter groupWriter =
tryToInitialGroupWriter(new PlainDeviceID(tablet.deviceId), isAligned);
- Path devicePath = new Path(tablet.deviceId, "", false);
+ Path devicePath = new Path(tablet.deviceId);
List<MeasurementSchema> schemas = tablet.getSchemas();
if (schema.containsDevice(devicePath)) {
checkIsAllMeasurementsInGroup(schema.getSeriesSchema(devicePath),
schemas, isAligned);
diff --git a/tsfile/src/test/java/org/apache/tsfile/write/PerformanceTest.java
b/tsfile/src/test/java/org/apache/tsfile/write/PerformanceTest.java
deleted file mode 100644
index 2b820c52..00000000
--- a/tsfile/src/test/java/org/apache/tsfile/write/PerformanceTest.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tsfile.write;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.exception.write.WriteProcessException;
-import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.PlainDeviceID;
-import org.apache.tsfile.file.metadata.enums.CompressionType;
-import org.apache.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.tsfile.read.TsFileSequenceReader;
-import org.apache.tsfile.read.common.Path;
-import org.apache.tsfile.read.controller.CachedChunkLoaderImpl;
-import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl;
-import org.apache.tsfile.read.expression.QueryExpression;
-import org.apache.tsfile.read.query.dataset.QueryDataSet;
-import org.apache.tsfile.read.query.executor.QueryExecutor;
-import org.apache.tsfile.read.query.executor.TsFileExecutor;
-import org.apache.tsfile.write.record.Tablet;
-import org.apache.tsfile.write.schema.IMeasurementSchema;
-import org.apache.tsfile.write.schema.MeasurementSchema;
-
-public class PerformanceTest {
-
- private final String testDir = "target" + File.separator + "tableViewTest";
- private final int idSchemaCnt = 3;
- private final int measurementSchemaCnt = 100;
- private final int tableCnt = 100;
- private final int devicePerTable = 10;
- private final int pointPerSeries = 100;
- private final int tabletCnt = 10;
-
- private List<MeasurementSchema> measurementSchemas;
-
- private List<Long> registerTimeList = new ArrayList<>();
- private List<Long> writeTimeList = new ArrayList<>();
- private List<Long> closeTimeList = new ArrayList<>();
- private List<Long> queryTimeList = new ArrayList<>();
- private List<Long> fileSizeList = new ArrayList<>();
-
- public static void main(String[] args) throws Exception{
- final PerformanceTest test = new PerformanceTest();
- test.initSchemas();
-
- int repetitionCnt = 10;
- for (int i = 0; i < repetitionCnt; i++) {
- test.testTree();
- }
-
- final double registerTime = test.registerTimeList.subList(repetitionCnt /
2, repetitionCnt).stream()
- .mapToLong(l -> l).average().orElse(
- 0.0f);
- final double writeTime = test.writeTimeList.subList(repetitionCnt / 2,
repetitionCnt).stream()
- .mapToLong(l -> l).average().orElse(
- 0.0f);
- final double closeTime = test.closeTimeList.subList(repetitionCnt / 2,
repetitionCnt).stream()
- .mapToLong(l -> l).average().orElse(
- 0.0f);
- final double queryTime = test.queryTimeList.subList(repetitionCnt / 2,
repetitionCnt).stream()
- .mapToLong(l -> l).average().orElse(
- 0.0f);
- final double fileSize = test.fileSizeList.subList(repetitionCnt / 2,
repetitionCnt).stream()
- .mapToLong(l -> l).average().orElse(
- 0.0f);
- System.out.printf("Register %fns, write %fns, close %fns, query %fns,
fileSize %f %n",
- registerTime,
- writeTime, closeTime, queryTime, fileSize);
- }
-
- private void initSchemas() {
- measurementSchemas = new ArrayList<>();
- for (int i = 0; i < measurementSchemaCnt; i++) {
- measurementSchemas.add(new MeasurementSchema(
- "s" + i, TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4));
- }
- }
-
- private void testTree() throws IOException, WriteProcessException {
- long registerTimeSum = 0;
- long writeTimeSum = 0;
- long closeTimeSum = 0;
- long queryTimeSum = 0;
- long startTime;
- final File file = initFile();
- TsFileWriter tsFileWriter = new TsFileWriter(file);
- try {
- startTime = System.nanoTime();
- registerTree(tsFileWriter);
- registerTimeSum = System.nanoTime() - startTime;
- Tablet tablet = initTreeTablet();
- for (int tableNum = 0; tableNum < tableCnt; tableNum++) {
- for (int deviceNum = 0; deviceNum < devicePerTable; deviceNum++) {
- for (int tabletNum = 0; tabletNum < tabletCnt; tabletNum++) {
- fillTreeTablet(tablet, tableNum, deviceNum, tabletNum);
- startTime = System.nanoTime();
- tsFileWriter.writeAligned(tablet);
- writeTimeSum += System.nanoTime() - startTime;
- }
- }
- }
- } finally {
- startTime = System.nanoTime();
- tsFileWriter.close();
- closeTimeSum = System.nanoTime() - startTime;
- }
- long fileSize = file.length();
-
- startTime = System.nanoTime();
- try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(file.getAbsolutePath())) {
- QueryExecutor queryExecutor =
- new TsFileExecutor(
- new MetadataQuerierByFileImpl(sequenceReader),
- new CachedChunkLoaderImpl(sequenceReader));
-
- List<Path> selectedSeries = new ArrayList<>();
- for (int i = 0; i < measurementSchemaCnt; i++) {
- for (int j = 0; j < devicePerTable; j++) {
- selectedSeries.add(new Path(genTreeDeviceId(tableCnt / 2,
j).getDevice(), "s" + i,
- false));
- }
- }
- final QueryExpression queryExpression =
QueryExpression.create(selectedSeries, null);
- final QueryDataSet queryDataSet = queryExecutor.execute(queryExpression);
- int cnt = 0;
- while (queryDataSet.hasNext()) {
- queryDataSet.next();
- cnt++;
- }
- }
- queryTimeSum = System.nanoTime() - startTime;
- file.delete();
-
- System.out.printf("Tree register %dns, write %dns, close %dns, query %dns,
fileSize %d %n",
- registerTimeSum,
- writeTimeSum, closeTimeSum, queryTimeSum, fileSize);
- registerTimeList.add(registerTimeSum);
- writeTimeList.add(writeTimeSum);
- closeTimeList.add(closeTimeSum);
- queryTimeList.add(queryTimeSum);
- fileSizeList.add(fileSize);
-
- }
-
- private File initFile() throws IOException {
- File dir = new File(testDir);
- dir.mkdirs();
- return new File(dir, "testTsFile");
- }
-
- private Tablet initTreeTablet() {
- return new Tablet(null, measurementSchemas, pointPerSeries);
- }
-
- private void fillTreeTablet(Tablet tablet, int tableNum, int deviceNum, int
tabletNum) {
- tablet.deviceId = genTreeDeviceId(tableNum, deviceNum).getDevice();
- for (int i = 0; i < measurementSchemaCnt; i++) {
- long[] values = (long[]) tablet.values[i];
- for (int valNum = 0; valNum < pointPerSeries; valNum++) {
- values[valNum] = (long) tabletNum * pointPerSeries + valNum;
- }
- }
- for (int valNum = 0; valNum < pointPerSeries; valNum++) {
- tablet.timestamps[valNum] = (long) tabletNum * pointPerSeries + valNum;
- }
- tablet.rowSize = pointPerSeries;
- }
-
- private void registerTree(TsFileWriter writer) throws WriteProcessException {
- for (int tableNum = 0; tableNum < tableCnt; tableNum++) {
- for (int deviceNum = 0; deviceNum < devicePerTable; deviceNum++) {
- writer.registerAlignedTimeseries(genTreeDeviceId(tableNum, deviceNum),
- measurementSchemas);
- }
- }
- }
-
- private IMeasurementSchema genMeasurementSchema(int measurementNum) {
- return measurementSchemas.get(measurementNum);
- }
-
- private String genTableName(int tableNum) {
- return "table_" + tableNum;
- }
-
- private String genDeviceId(int tableNum, int deviceNum) {
- String[] idSegments = new String[idSchemaCnt + 1];
- idSegments[0] = genTableName(tableNum);
- for (int i = 0; i < idSchemaCnt; i++) {
- idSegments[i + 1] = "0";
- }
- idSegments[idSchemaCnt] = Integer.toString(deviceNum);
- return String.join(".", idSegments);
- }
-
- private Path genTreeDeviceId(int tableNum, int deviceNum) {
- return new Path(genDeviceId(tableNum, deviceNum), "", false);
- }
-
-}
diff --git a/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java
b/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java
index 1091dac7..49bba8d7 100644
--- a/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java
+++ b/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriterTest.java
@@ -31,6 +31,7 @@ import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.read.expression.QueryExpression;
import org.apache.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.tsfile.utils.TsFileGeneratorForTest;
import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.record.datapoint.FloatDataPoint;
@@ -44,6 +45,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -55,7 +57,7 @@ import static org.junit.Assert.fail;
public class TsFileWriterTest {
TsFileWriter writer = null;
- String fileName = "target/v3TsFile";
+ String fileName = TsFileGeneratorForTest.getTestTsFilePath("root.sg1", 0, 0,
1);
boolean closed = false;
@Before
@@ -78,12 +80,12 @@ public class TsFileWriterTest {
if (!closed) {
closeFile();
}
- // try {
- // Files.deleteIfExists(new File(fileName).toPath());
- // } catch (IOException e) {
- // e.printStackTrace();
- // fail(e.getMessage());
- // }
+ try {
+ Files.deleteIfExists(new File(fileName).toPath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
}
private void registerTimeseries() {
