This is an automated email from the ASF dual-hosted git repository. yongzao pushed a commit to branch enhance-ain-CI in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e5cb3543115dae0b96fdb3ad5d32cddadf5838fc Author: Yongzao <[email protected]> AuthorDate: Tue Aug 5 15:33:19 2025 +0800 Enhance CI test envs update CI envs --- integration-test/src/assembly/mpp-test.xml | 5 - .../iotdb/it/env/cluster/ClusterConstant.java | 1 - .../iotdb/it/env/cluster/node/AINodeWrapper.java | 3 - .../iotdb/ainode/it/AINodeInferenceSQLIT.java | 176 +++++++++++---------- .../iotdb/ainode/it/AINodeModelManageIT.java | 17 +- .../apache/iotdb/ainode/utils/AINodeTestUtils.java | 3 +- iotdb-core/ainode/pom.xml | 16 -- .../ainode/resources/syncPythonVersion.groovy | 4 +- 8 files changed, 105 insertions(+), 120 deletions(-) diff --git a/integration-test/src/assembly/mpp-test.xml b/integration-test/src/assembly/mpp-test.xml index 71f184549b2..4bcd32c7ee0 100644 --- a/integration-test/src/assembly/mpp-test.xml +++ b/integration-test/src/assembly/mpp-test.xml @@ -61,11 +61,6 @@ <directory>${project.basedir}/../scripts/tools</directory> <fileMode>0755</fileMode> </fileSet> - <fileSet> - <outputDirectory>venv</outputDirectory> - <directory>${project.basedir}/../iotdb-core/ainode/venv</directory> - <fileMode>0755</fileMode> - </fileSet> <fileSet> <outputDirectory>lib</outputDirectory> <directory>${project.basedir}/../iotdb-core/ainode/dist/</directory> diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java index 3162f47ebe8..d6b3267989c 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java @@ -209,7 +209,6 @@ public class ClusterConstant { // Paths public static final String USER_DIR = "user.dir"; public static final String TARGET = "target"; - public static final String PYTHON_PATH = "venv/bin/python3"; public static final String DATA_NODE_NAME = "DataNode"; diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java index 608b799f7a8..64c2bbec2c0 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AINodeWrapper.java @@ -33,7 +33,6 @@ import java.util.ArrayList; import java.util.List; import static org.apache.iotdb.it.env.cluster.ClusterConstant.AI_NODE_NAME; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.PYTHON_PATH; import static org.apache.iotdb.it.env.cluster.ClusterConstant.TARGET; import static org.apache.iotdb.it.env.cluster.ClusterConstant.USER_DIR; import static org.apache.iotdb.it.env.cluster.EnvUtils.getTimeForLogDirectory; @@ -120,8 +119,6 @@ public class AINodeWrapper extends AbstractNodeWrapper { List<String> startCommand = new ArrayList<>(); startCommand.add(SHELL_COMMAND); startCommand.add(filePrefix + File.separator + SCRIPT_PATH + File.separator + SCRIPT_FILE); - startCommand.add("-i"); - startCommand.add(filePrefix + File.separator + PYTHON_PATH); startCommand.add("-r"); ProcessBuilder processBuilder = diff --git a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeInferenceSQLIT.java b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeInferenceSQLIT.java index 6064513ccc0..7eff4629eb9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeInferenceSQLIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeInferenceSQLIT.java @@ -26,6 +26,7 @@ import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -90,8 +91,7 @@ public class AINodeInferenceSQLIT { EnvFactory.getEnv().cleanClusterEnvironment(); } - // TODO: We need reconsider call inference before enable this IT - // @Test + @Test public void callInferenceTestInTree() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); Statement statement = connection.createStatement()) { @@ -121,44 +121,45 @@ public class AINodeInferenceSQLIT { // SQL4: built-in model inferences single column with given predict_length String sql4 = "CALL INFERENCE(holtwinters, \"select s0 from root.AI\", predict_length=6, generateTime=true)"; + // TODO: enable following tests after refactor the CALL INFERENCE - try (ResultSet resultSet = statement.executeQuery(sql1)) { - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - checkHeader(resultSetMetaData, "Time,output0,output1,output2,output3"); - int count = 0; - while (resultSet.next()) { - float s0 = resultSet.getFloat(2); - float s1 = resultSet.getFloat(3); - float s2 = resultSet.getFloat(4); - float s3 = resultSet.getFloat(5); - - assertEquals(s0, count + 1.0, 0.0001); - assertEquals(s1, count + 2.0, 0.0001); - assertEquals(s2, count + 3.0, 0.0001); - assertEquals(s3, count + 4.0, 0.0001); - count++; - } - assertEquals(7, count); - } - - try (ResultSet resultSet = statement.executeQuery(sql2)) { - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - checkHeader(resultSetMetaData, "output0,output1,output2"); - int count = 0; - while (resultSet.next()) { - float s2 = resultSet.getFloat(1); - float s0 = resultSet.getFloat(2); - float s3 = resultSet.getFloat(3); - float s1 = resultSet.getFloat(4); - - assertEquals(s0, count + 1.0, 0.0001); - assertEquals(s1, count + 2.0, 0.0001); - assertEquals(s2, count + 3.0, 0.0001); - assertEquals(s3, count + 4.0, 0.0001); - count++; - } - assertEquals(7, count); - } + // try (ResultSet resultSet = statement.executeQuery(sql1)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader(resultSetMetaData, "Time,output0,output1,output2,output3"); + // int count = 0; + // while (resultSet.next()) { + // float s0 = resultSet.getFloat(2); + // float s1 = resultSet.getFloat(3); + // float s2 = resultSet.getFloat(4); + // float s3 = resultSet.getFloat(5); + // + // assertEquals(s0, count + 1.0, 0.0001); + // assertEquals(s1, count + 2.0, 0.0001); + // assertEquals(s2, count + 3.0, 0.0001); + // assertEquals(s3, count + 4.0, 0.0001); + // count++; + // } + // assertEquals(7, count); + // } + // + // try (ResultSet resultSet = statement.executeQuery(sql2)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader(resultSetMetaData, "output0,output1,output2"); + // int count = 0; + // while (resultSet.next()) { + // float s2 = resultSet.getFloat(1); + // float s0 = resultSet.getFloat(2); + // float s3 = resultSet.getFloat(3); + // float s1 = resultSet.getFloat(4); + // + // assertEquals(s0, count + 1.0, 0.0001); + // assertEquals(s1, count + 2.0, 0.0001); + // assertEquals(s2, count + 3.0, 0.0001); + // assertEquals(s3, count + 4.0, 0.0001); + // count++; + // } + // assertEquals(7, count); + // } try (ResultSet resultSet = statement.executeQuery(sql3)) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); @@ -181,8 +182,7 @@ public class AINodeInferenceSQLIT { } } - // TODO: We need reconsider call inference before enable this IT - // @Test + @Test public void errorCallInferenceTestInTree() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); Statement statement = connection.createStatement()) { @@ -203,18 +203,19 @@ public class AINodeInferenceSQLIT { String sql = "CALL INFERENCE(notFound404, \"select s0,s1,s2 from root.AI\", window=head(5))"; errorTest(statement, sql, "1505: model [notFound404] has not been created."); sql = "CALL INFERENCE(identity, \"select s0,s1,s2,s3 from root.AI\", window=head(2))"; - errorTest(statement, sql, "701: Window output 2 is not equal to input size of model 7"); + // TODO: enable following tests after refactor the CALL INFERENCE + // errorTest(statement, sql, "701: Window output 2 is not equal to input size of model 7"); sql = "CALL INFERENCE(identity, \"select s0,s1,s2,s3 from root.AI limit 5\")"; - errorTest( - statement, - sql, - "301: The number of rows 5 in the input data does not match the model input 7. Try to use LIMIT in SQL or WINDOW in CALL INFERENCE"); + // errorTest( + // statement, + // sql, + // "301: The number of rows 5 in the input data does not match the model input 7. Try to + // use LIMIT in SQL or WINDOW in CALL INFERENCE"); sql = "CREATE MODEL 中文 USING URI \"" + EXAMPLE_MODEL_PATH + "\""; errorTest(statement, sql, "701: ModelId can only contain letters, numbers, and underscores"); } - // TODO: Our function is too bad currently - // @Test + @Test public void selectForecastTestInTable() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { @@ -230,47 +231,48 @@ public class AINodeInferenceSQLIT { // SQL4: built-in model inferences single column with given predict_length String sql4 = "SELECT * FROM FORECAST(model_id=>'holtwinters', input=>(SELECT time,s0 FROM root.AI) ORDER BY time, output_length=>6)"; - try (ResultSet resultSet = statement.executeQuery(sql1)) { - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - checkHeader(resultSetMetaData, "time,s0,s1,s2,s3"); - int count = 0; - while (resultSet.next()) { - float s0 = resultSet.getFloat(2); - float s1 = resultSet.getFloat(3); - float s2 = resultSet.getFloat(4); - float s3 = resultSet.getFloat(5); - - assertEquals(s0, count + 1.0, 0.0001); - assertEquals(s1, count + 2.0, 0.0001); - assertEquals(s2, count + 3.0, 0.0001); - assertEquals(s3, count + 4.0, 0.0001); - count++; - } - assertEquals(7, count); - } - - try (ResultSet resultSet = statement.executeQuery(sql2)) { - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - checkHeader(resultSetMetaData, "time,s2,s0,s3,s1"); - int count = 0; - while (resultSet.next()) { - float s2 = resultSet.getFloat(1); - float s0 = resultSet.getFloat(2); - float s3 = resultSet.getFloat(3); - float s1 = resultSet.getFloat(4); - - assertEquals(s0, count + 1.0, 0.0001); - assertEquals(s1, count + 2.0, 0.0001); - assertEquals(s2, count + 3.0, 0.0001); - assertEquals(s3, count + 4.0, 0.0001); - count++; - } - assertEquals(7, count); - } + // TODO: enable following tests after refactor the FORECAST + // try (ResultSet resultSet = statement.executeQuery(sql1)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader(resultSetMetaData, "time,s0,s1,s2,s3"); + // int count = 0; + // while (resultSet.next()) { + // float s0 = resultSet.getFloat(2); + // float s1 = resultSet.getFloat(3); + // float s2 = resultSet.getFloat(4); + // float s3 = resultSet.getFloat(5); + // + // assertEquals(s0, count + 1.0, 0.0001); + // assertEquals(s1, count + 2.0, 0.0001); + // assertEquals(s2, count + 3.0, 0.0001); + // assertEquals(s3, count + 4.0, 0.0001); + // count++; + // } + // assertEquals(7, count); + // } + // + // try (ResultSet resultSet = statement.executeQuery(sql2)) { + // ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // checkHeader(resultSetMetaData, "time,s2,s0,s3,s1"); + // int count = 0; + // while (resultSet.next()) { + // float s2 = resultSet.getFloat(1); + // float s0 = resultSet.getFloat(2); + // float s3 = resultSet.getFloat(3); + // float s1 = resultSet.getFloat(4); + // + // assertEquals(s0, count + 1.0, 0.0001); + // assertEquals(s1, count + 2.0, 0.0001); + // assertEquals(s2, count + 3.0, 0.0001); + // assertEquals(s3, count + 4.0, 0.0001); + // count++; + // } + // assertEquals(7, count); + // } try (ResultSet resultSet = statement.executeQuery(sql3)) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - checkHeader(resultSetMetaData, "time,output0,output1,output2"); + checkHeader(resultSetMetaData, "time,s0,s1,s2"); int count = 0; while (resultSet.next()) { count++; @@ -280,7 +282,7 @@ public class AINodeInferenceSQLIT { try (ResultSet resultSet = statement.executeQuery(sql4)) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - checkHeader(resultSetMetaData, "time,output0"); + checkHeader(resultSetMetaData, "time,s0"); int count = 0; while (resultSet.next()) { count++; diff --git a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeModelManageIT.java b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeModelManageIT.java index f7220f20cbe..af6681fc9b8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeModelManageIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeModelManageIT.java @@ -38,6 +38,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.AbstractMap; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -45,6 +46,7 @@ import static org.apache.iotdb.ainode.utils.AINodeTestUtils.EXAMPLE_MODEL_PATH; import static org.apache.iotdb.ainode.utils.AINodeTestUtils.checkHeader; import static org.apache.iotdb.ainode.utils.AINodeTestUtils.errorTest; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -96,7 +98,7 @@ public class AINodeModelManageIT { } @Test - public void userDefinedModelManagementTestInTree() throws SQLException { + public void userDefinedModelManagementTestInTree() throws SQLException, InterruptedException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); Statement statement = connection.createStatement()) { userDefinedModelManagementTest(statement); @@ -104,14 +106,15 @@ public class AINodeModelManageIT { } @Test - public void userDefinedModelManagementTestInTable() throws SQLException { + public void userDefinedModelManagementTestInTable() throws SQLException, InterruptedException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { userDefinedModelManagementTest(statement); } } - private void userDefinedModelManagementTest(Statement statement) throws SQLException { + private void userDefinedModelManagementTest(Statement statement) + throws SQLException, InterruptedException { final String alterConfigSQL = "set configuration \"trusted_uri_pattern\"='.*'"; final String registerSql = "create model operationTest using uri \"" + EXAMPLE_MODEL_PATH + "\""; @@ -122,7 +125,7 @@ public class AINodeModelManageIT { statement.execute(registerSql); boolean loading = true; int count = 0; - while (loading) { + for (int retryCnt = 0; retryCnt < 100; retryCnt++) { try (ResultSet resultSet = statement.executeQuery(showSql)) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); checkHeader(resultSetMetaData, "ModelId,ModelType,Category,State"); @@ -130,7 +133,6 @@ public class AINodeModelManageIT { String modelId = resultSet.getString(1); String category = resultSet.getString(3); String state = resultSet.getString(4); - assertEquals("operationTest", modelId); assertEquals("USER-DEFINED", category); if (state.equals("ACTIVE")) { @@ -143,7 +145,12 @@ public class AINodeModelManageIT { } } } + if (!loading) { + break; // Model is loaded successfully + } + TimeUnit.SECONDS.sleep(1); } + assertFalse(loading); assertEquals(1, count); statement.execute(dropSql); try (ResultSet resultSet = statement.executeQuery(showSql)) { diff --git a/integration-test/src/test/java/org/apache/iotdb/ainode/utils/AINodeTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/ainode/utils/AINodeTestUtils.java index e9464827c79..d3f36e38efe 100644 --- a/integration-test/src/test/java/org/apache/iotdb/ainode/utils/AINodeTestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/ainode/utils/AINodeTestUtils.java @@ -32,7 +32,8 @@ import static org.junit.Assert.fail; public class AINodeTestUtils { public static final String EXAMPLE_MODEL_PATH = - System.getProperty("user.dir") + "file://" + + System.getProperty("user.dir") + File.separator + "src" + File.separator diff --git a/iotdb-core/ainode/pom.xml b/iotdb-core/ainode/pom.xml index bcd2821644b..c3218d2c99f 100644 --- a/iotdb-core/ainode/pom.xml +++ b/iotdb-core/ainode/pom.xml @@ -260,22 +260,6 @@ </arguments> </configuration> </execution> - <!-- Refoemat code --> - <!--execution> - <id>python-black</id> - <phase>process-sources</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>${python.venv.bin}${python.exe.bin}</executable> - <arguments> - <argument>-m</argument> - <argument>black</argument> - <argument>.</argument> - </arguments> - </configuration> - </execution--> <execution> <id>python-compile</id> <phase>compile</phase> diff --git a/iotdb-core/ainode/resources/syncPythonVersion.groovy b/iotdb-core/ainode/resources/syncPythonVersion.groovy index 373bfd7fa0a..0061930bfca 100644 --- a/iotdb-core/ainode/resources/syncPythonVersion.groovy +++ b/iotdb-core/ainode/resources/syncPythonVersion.groovy @@ -99,7 +99,7 @@ def checkPython() { Matcher matcher = extractVersion(stdOut + stdErr) if (matcher.size() > 0) { String curVersion = matcher[0][1] - def result = checkVersionAtLeast(curVersion, "3.8.0") + def result = checkVersionAtLeast(curVersion, "3.9.0") if (!result) { allConditionsMet = false } @@ -108,7 +108,7 @@ def checkPython() { allConditionsMet = false } } else { - println "missing (Please install at least version 3.8.0 and at most one of the 3.13.x versions)" + println "missing (Please install at least version 3.9.0 and at most one of the 3.13.x versions)" allConditionsMet = false } } catch (Exception ignored) {
