This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/area-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 24d39c386f08dd27485461763df4b2c6f7881862 Author: Lei Rui <[email protected]> AuthorDate: Mon Feb 3 21:15:24 2025 +0800 add --- server/pom.xml | 74 ++++++------ .../resources/conf/iotdb-engine.properties | 16 +-- .../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 5 + .../db/query/udf/builtin/BuiltinFunction.java | 3 +- .../iotdb/db/query/udf/builtin/UDTFEBUG.java | 126 +++++++++++++++++++ .../apache/iotdb/db/integration/eBUG/MyTest.java | 133 +++++++++++++++++++++ 6 files changed, 311 insertions(+), 46 deletions(-) diff --git a/server/pom.xml b/server/pom.xml index efa727e5732..1688dc9e4c9 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -284,43 +284,43 @@ </execution> </executions> </plugin> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <finalName>sample_ltd</finalName> - <!-- <finalName>sample_swab_ad</finalName>--> - <!-- <finalName>sample_swab</finalName>--> - <!-- <finalName>sample_rdp</finalName>--> - <!-- <finalName>sample_minmax</finalName>--> - <!-- <finalName>sample_fsw</finalName>--> - <!-- <finalName>sample_eBUG</finalName>--> - <!-- <finalName>sample_BUYdiff</finalName>--> - <archive> - <manifest> - <mainClass>org.apache.iotdb.db.query.eBUG.sample_LTD</mainClass> - <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWABAD</mainClass>--> - <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>--> - <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>--> - <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>--> - <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>--> - <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>--> - <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>--> - </manifest> - </archive> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - </configuration> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> + <!-- <plugin>--> + <!-- <artifactId>maven-assembly-plugin</artifactId>--> + <!-- <configuration>--> + <!-- <finalName>sample_ltd</finalName>--> + <!-- <!– <finalName>sample_swab_ad</finalName>–>--> + <!-- <!– <finalName>sample_swab</finalName>–>--> + <!-- <!– <finalName>sample_rdp</finalName>–>--> + <!-- <!– <finalName>sample_minmax</finalName>–>--> + <!-- <!– <finalName>sample_fsw</finalName>–>--> + <!-- <!– <finalName>sample_eBUG</finalName>–>--> + <!-- <!– <finalName>sample_BUYdiff</finalName>–>--> + <!-- <archive>--> + <!-- <manifest>--> + <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_LTD</mainClass>--> + <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWABAD</mainClass>–>--> + <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>–>--> + <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>–>--> + <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>–>--> + <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>–>--> + <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>–>--> + <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>–>--> + <!-- </manifest>--> + <!-- </archive>--> + <!-- <descriptorRefs>--> + <!-- <descriptorRef>jar-with-dependencies</descriptorRef>--> + <!-- </descriptorRefs>--> + <!-- </configuration>--> + <!-- <executions>--> + <!-- <execution>--> + <!-- <id>make-assembly</id>--> + <!-- <phase>package</phase>--> + <!-- <goals>--> + <!-- <goal>single</goal>--> + <!-- </goals>--> + <!-- </execution>--> + <!-- </executions>--> + <!-- </plugin>--> </plugins> </build> <profiles> diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index d68392e0e9d..f89417c5430 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -38,10 +38,10 @@ pnv=100 #for ILTS numIterations=4 -acc_avg=true -acc_rectangle=true -acc_convex=true -acc_iterRepeat=true +acc_avg=false +acc_rectangle=false +acc_convex=false +acc_iterRepeat=false #################### ### enable CPV @@ -275,7 +275,7 @@ seq_tsfile_size=1073741824 # close_tsfile_check_interval_in_ms=600000 # When the average point number of timeseries in memtable exceeds this, the memtable is flushed to disk. The default threshold is 10000. -avg_series_point_number_threshold=100 +avg_series_point_number_threshold=10000 # How many threads can concurrently flush. When <= 0, use CPU core number. # concurrent_flush_thread=0 @@ -632,17 +632,17 @@ enable_unseq_compaction=false ### Configurations for tsfile-format #################### -write_convex_hull=true +write_convex_hull=false enableMinMaxLSM=false use_Statistics=true -use_TimeIndex=true +use_TimeIndex=false use_Mad=false -use_ValueIndex=true +use_ValueIndex=false error_Param=1 diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java index 44fc3298d96..86d9abaccde 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java +++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java @@ -353,6 +353,11 @@ public class eBUG { // return results; // 注意这就是lineToSimplify.getVertices() resultsBottomUpEliminated.add(points.get(0)); // 全局首点 resultsBottomUpEliminated.add(points.get(points.size() - 1)); // 全局尾点 + + // make them ordered by dominated sig in descending order (i.e., reverse of bottom-up + // elimination order) + Collections.reverse(resultsBottomUpEliminated); // O(n) + return resultsBottomUpEliminated; } } diff --git a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java index 5a9639faaf4..cc51a486e56 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java +++ b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java @@ -50,7 +50,8 @@ public enum BuiltinFunction { M4("M4", UDTFM4MAC.class), M4_TW("M4_TW", UDTFM4.class), Sample("SAMPLE", UDTFSample.class), - MinMax("MINMAX", UDTFMinMax.class); + MinMax("MINMAX", UDTFMinMax.class), + EBUG("EBUG", UDTFEBUG.class), ; private final String functionName; diff --git a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFEBUG.java b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFEBUG.java new file mode 100644 index 00000000000..011ae9a7655 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFEBUG.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.query.udf.builtin; + +import org.apache.iotdb.db.exception.metadata.MetadataException; +import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.query.eBUG.Point; +import org.apache.iotdb.db.query.eBUG.eBUG; +import org.apache.iotdb.db.query.udf.api.UDTF; +import org.apache.iotdb.db.query.udf.api.access.Row; +import org.apache.iotdb.db.query.udf.api.collector.PointCollector; +import org.apache.iotdb.db.query.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.db.query.udf.api.customizer.parameter.UDFParameterValidator; +import org.apache.iotdb.db.query.udf.api.customizer.parameter.UDFParameters; +import org.apache.iotdb.db.query.udf.api.customizer.strategy.RowByRowAccessStrategy; +import org.apache.iotdb.db.query.udf.api.exception.UDFException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class UDTFEBUG implements UDTF { + + protected TSDataType dataType; + + private int m; // target number of sampled points + + private int e; + + private List<Point> points; + + @Override + public void validate(UDFParameterValidator validator) throws UDFException { + validator + .validateInputSeriesNumber(1) + .validateInputSeriesDataType( + 0, TSDataType.INT32, TSDataType.INT64, TSDataType.FLOAT, TSDataType.DOUBLE) + .validateRequiredAttribute("m"); + } + + @Override + public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) + throws MetadataException { + dataType = parameters.getDataType(0); + m = parameters.getInt("m"); // m>2 for online sampling mode, <=2 for precomputation mode + // if (m <= 2) { + // throw new MetadataException("m should be larger than 2 for online sampling mode"); + // } + e = parameters.getInt("e"); + if (e < 0) { + throw new MetadataException("e should not be smaller than 0"); + } + + if (m <= 2) { // precomputation mode + configurations + .setAccessStrategy(new RowByRowAccessStrategy()) + .setOutputDataType(TSDataType.TEXT); + } else { // online sampling mode + configurations + .setAccessStrategy(new RowByRowAccessStrategy()) + .setOutputDataType(TSDataType.DOUBLE); + } + points = new ArrayList<>(); + } + + @Override + public void transform(Row row, PointCollector collector) + throws QueryProcessException, IOException { + double t = row.getTime(); + switch (dataType) { + case INT32: + points.add(new Point(t, row.getInt(0))); + break; + case INT64: + points.add(new Point(t, row.getLong(0))); + break; + case FLOAT: + points.add(new Point(t, row.getFloat(0))); + break; + case DOUBLE: + points.add(new Point(t, row.getDouble(0))); + break; + default: + break; + } + } + + @Override + public void terminate(PointCollector collector) throws IOException, QueryProcessException { + List<Point> sampled = eBUG.buildEffectiveArea(points, e, false, m); + + // collect result + if (m <= 2) { + // offline precomputation mode, for precomputing the dominated significance of each, ordered + // by dominated sig in descending order (i.e., reverse of bottom-up elimination order) + for (int i = 0; i < sampled.size(); i++) { + collector.putString(i + 1, sampled.get(i).toString()); + // not using dominated significance z as timestamp, + // because dominated significance z can be double, if converted to long may lose some order + // information + } + } else { + // m>2, online sampling mode, order by time in ascending order + for (Point p : sampled) { + collector.putDouble((long) p.getTimestamp(), p.getValue()); + } + } + } +} diff --git a/server/src/test/java/org/apache/iotdb/db/integration/eBUG/MyTest.java b/server/src/test/java/org/apache/iotdb/db/integration/eBUG/MyTest.java new file mode 100644 index 00000000000..e276ce47143 --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/integration/eBUG/MyTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.integration.eBUG; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.CompactionStrategy; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.jdbc.Config; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Locale; + +import static org.junit.Assert.fail; + +public class MyTest { + + private static final String TIMESTAMP_STR = "Time"; + + private static String[] creationSqls = + new String[] { + "SET STORAGE GROUP TO root.vehicle.d0", + "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=DOUBLE,ENCODING=PLAIN", + }; + + private final String d0s0 = "root.vehicle.d0.s0"; + + private static final String insertTemplate = + "INSERT INTO root.vehicle.d0(timestamp,s0)" + " VALUES(%d,%f)"; + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static boolean originalEnableCPV; + private static CompactionStrategy originalCompactionStrategy; + private static boolean originalUseChunkIndex; + + @Before + public void setUp() throws Exception { + TSFileDescriptor.getInstance().getConfig().setTimeEncoder("PLAIN"); + originalCompactionStrategy = config.getCompactionStrategy(); + config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION); + + EnvironmentUtils.envSetUp(); + Class.forName(Config.JDBC_DRIVER_NAME); + // config.setTimestampPrecision("ms"); + } + + @After + public void tearDown() throws Exception { + EnvironmentUtils.cleanEnv(); + config.setCompactionStrategy(originalCompactionStrategy); + } + + @Test + public void test1() { + prepareData1(); + + // String[] res = + // new String[]{ + // "0,1,20,5.0,20.0,5.0[1],30.0[10]", + // "25,25,45,8.0,30.0,8.0[25],40.0[30]", + // "50,52,54,8.0,18.0,8.0[52],18.0[54]", + // "75,null,null,null,null,null,null" + // }; + try (Connection connection = + DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = + statement.execute("SELECT EBUG(s0,'m'='10','e'='0') FROM root.vehicle.d0"); + + Assert.assertTrue(hasResultSet); + try (ResultSet resultSet = statement.getResultSet()) { + int i = 0; + while (resultSet.next()) { + String ans = resultSet.getString(1) + "," + resultSet.getString(2); + System.out.println(ans); + // Assert.assertEquals(res[i++], ans); + } + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private static void prepareData1() { + try (Connection connection = + DriverManager.getConnection( + Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + + for (String sql : creationSqls) { + statement.execute(sql); + } + + for (int i = 0; i < 1000; i++) { + double x = 2 * Math.PI * i / 1000.0; + double sineValue = Math.sin(x); + statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, sineValue)); + } + + statement.execute("FLUSH"); + + } catch (Exception e) { + e.printStackTrace(); + } + } +}
