This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch fix-copy-to-duplicate-tag-column in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5f8c9232c9bae09f23a2c2e03d07d0b729061ccb Author: shuwenwei <[email protected]> AuthorDate: Thu Aug 6 10:48:42 2026 +0800 Fix silent deduplication of duplicate tag columns in COPY TO statement Previously, TAGS(device, device) in a COPY TO statement was silently deduplicated to [device] because the identifiers were added to a LinkedHashSet, so no error was reported and the file was generated with fewer tag columns than the user specified. The duplicate information was lost before CopyToTsFileOptions.check() could validate it. Also, duplicate option clauses (e.g. (tags(a), tags(b)) or (time x, time y)) were silently overwritten by the last one in the CopyToOptions.Builder. Fix both cases at parse time in AstBuilder: - Detect duplicate tag columns in the TAGS clause and report "Duplicate tag column in TAGS clause: %s". - Track already-seen option keys and report "Duplicate option in COPY TO statement: %s". Add ITs covering both scenarios. --- .../query/recent/copyto/IoTDBCopyToTsFileIT.java | 32 ++++++++++++++++++++++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 4 +++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 4 +++ .../plan/relational/sql/parser/AstBuilder.java | 31 ++++++++++++++++++++- 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java index 04bed8b91a1..8e7cf274051 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java @@ -276,6 +276,38 @@ public class IoTDBCopyToTsFileIT { } } + @Test + public void testDuplicateTagColumns() + throws IoTDBConnectionException, StatementExecutionException, IOException { + try (ITableSession session = + EnvFactory.getEnv().getTableSessionConnectionWithDB(DATABASE_NAME)) { + try { + session.executeQueryStatement( + "copy table1(time,tag1,tag2,s1) to 'dup.tsfile' with (tags(tag1,tag1), memory_threshold 1000000)"); + Assert.fail("Should report duplicate tag column error"); + } catch (StatementExecutionException e) { + Assert.assertTrue( + e.getMessage(), e.getMessage().contains("Duplicate tag column in TAGS clause: tag1")); + } + } + } + + @Test + public void testDuplicateOption() + throws IoTDBConnectionException, StatementExecutionException, IOException { + try (ITableSession session = + EnvFactory.getEnv().getTableSessionConnectionWithDB(DATABASE_NAME)) { + try { + session.executeQueryStatement( + "copy table1(time,tag1,tag2,s1) to 'dup_option.tsfile' with (tags(tag1), tags(tag2), memory_threshold 1000000)"); + Assert.fail("Should report duplicate option error"); + } catch (StatementExecutionException e) { + Assert.assertTrue( + e.getMessage(), e.getMessage().contains("Duplicate option in COPY TO statement: TAGS")); + } + } + } + @Test public void testCopyWithSpecifiedTag() throws IoTDBConnectionException, StatementExecutionException, IOException { diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 40e7ed020c3..0e9e8158ac7 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -2424,6 +2424,10 @@ public final class DataNodeQueryMessages { "Note that the syntax for 'set configuration' in the tree model is not exactly the same as that in the table model."; public static final String UNSUPPORTED_COPY_TO_FORMAT_S_SUPPORTED_FORMATS_S = "Unsupported COPY TO format '%s'. Supported formats: %s"; + public static final String EXCEPTION_DUPLICATE_TAG_COLUMN_IN_TAGS_CLAUSE_ARG_61FD5422 = + "Duplicate tag column in TAGS clause: %s"; + public static final String EXCEPTION_DUPLICATE_OPTION_IN_COPY_TO_STATEMENT_ARG_99CFE09F = + "Duplicate option in COPY TO statement: %s"; public static final String SIMULTANEOUS_SETTING_OF_MONTHLY_AND_NON_MONTHLY_INTERVALS_IS_NOT_SUPPORTED = "Simultaneous setting of monthly and non-monthly intervals is not supported."; public static final String DON_T_NEED_TO_SPECIFY_TIME_COLUMN_WHILE_EITHER_TIME_BOUND_OR_FILL_GROUP_PARAMETER_IS_NOT = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index f236a57f6a4..1ab922fa121 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -2799,6 +2799,10 @@ public final class DataNodeQueryMessages { public static final String UNSUPPORTED_COPY_TO_FORMAT_S_SUPPORTED_FORMATS_S = "不支持 COPY_TO 格式 '%s'。支持的格式：%s"; + public static final String EXCEPTION_DUPLICATE_TAG_COLUMN_IN_TAGS_CLAUSE_ARG_61FD5422 = + "TAGS 子句中存在重复的 TAG 列：%s"; + public static final String EXCEPTION_DUPLICATE_OPTION_IN_COPY_TO_STATEMENT_ARG_99CFE09F = + "COPY TO 语句中存在重复的选项：%s"; public static final String SIMULTANEOUS_SETTING_OF_MONTHLY_AND_NON_MONTHLY_INTERVALS_IS_NOT_SUPPORTED = "不支持同时设置月级和非月级时间间隔。"; public static final String DON_T_NEED_TO_SPECIFY_TIME_COLUMN_WHILE_EITHER_TIME_BOUND_OR_FILL_GROUP_PARAMETER_IS_NOT = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 7db81826ab6..87e1b417e44 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -2270,8 +2270,17 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> { String targetFileName = parseStringLiteral(ctx.fileName.getText()); CopyToOptions.Builder copyToOptionsBuilder = new CopyToOptions.Builder(); if (optionsContext != null) { + Set<String> optionKeys = new HashSet<>(); for (RelationalSqlParser.CopyToStatementOptionContext context : optionsContext.copyToStatementOption()) { + String optionKey = getCopyToOptionKey(context); + if (!optionKeys.add(optionKey)) { + throw new SemanticException( + String.format( + DataNodeQueryMessages + .EXCEPTION_DUPLICATE_OPTION_IN_COPY_TO_STATEMENT_ARG_99CFE09F, + optionKey)); + } addCopyToOption(copyToOptionsBuilder, context); } } @@ -2296,6 +2305,20 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> { return new CopyTo(queryNode, targetFileName, copyToOptionsBuilder.build()); } + private String getCopyToOptionKey(RelationalSqlParser.CopyToStatementOptionContext context) { + if (context.FORMAT() != null) { + return "FORMAT"; + } else if (context.TABLE() != null) { + return "TABLE"; + } else if (context.TIME() != null) { + return "TIME"; + } else if (context.TAGS() != null) { + return "TAGS"; + } else { + return "MEMORY_THRESHOLD"; + } + } + private void addCopyToOption( CopyToOptions.Builder builder, RelationalSqlParser.CopyToStatementOptionContext context) { if (context.FORMAT() != null) { @@ -2325,7 +2348,13 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> { context.identifierList().identifier(); Set<String> targetTagColumns = new LinkedHashSet<>(identifierList.size()); for (RelationalSqlParser.IdentifierContext identifierContext : identifierList) { - targetTagColumns.add(((Identifier) visit(identifierContext)).getValue()); + String tagColumnName = ((Identifier) visit(identifierContext)).getValue(); + if (!targetTagColumns.add(tagColumnName)) { + throw new SemanticException( + String.format( + DataNodeQueryMessages.EXCEPTION_DUPLICATE_TAG_COLUMN_IN_TAGS_CLAUSE_ARG_61FD5422, + tagColumnName)); + } } builder.withTargetTagColumns(targetTagColumns); } else if (context.MEMORY_THRESHOLD() != null) {
