This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 513a093f6dc Fix silent deduplication of duplicate tag columns in COPY
TO statement (#18408)
513a093f6dc is described below
commit 513a093f6dc4f691c900c4a28f8afafaedb7b115
Author: shuwenwei <[email protected]>
AuthorDate: Thu Aug 6 14:49:16 2026 +0800
Fix silent deduplication of duplicate tag columns in COPY TO statement
(#18408)
---
.../query/recent/copyto/IoTDBCopyToTsFileIT.java | 38 ++++++++++++++++++++--
.../iotdb/db/i18n/DataNodeQueryMessages.java | 4 +++
.../iotdb/db/i18n/DataNodeQueryMessages.java | 4 +++
.../process/copyto/tsfile/CopyToTsFileOptions.java | 13 ++++++--
.../copyto/tsfile/TsFileFormatCopyToWriter.java | 6 +++-
.../plan/relational/sql/parser/AstBuilder.java | 31 +++++++++++++++++-
.../schema/column/ColumnHeaderConstant.java | 2 +-
7 files changed, 90 insertions(+), 8 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java
index 04bed8b91a1..e25c3f58299 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/copyto/IoTDBCopyToTsFileIT.java
@@ -276,6 +276,38 @@ public class IoTDBCopyToTsFileIT {
}
}
+ @Test
+ public void testDuplicateTagColumns()
+ throws IoTDBConnectionException, StatementExecutionException,
IOException {
+ try (ITableSession session =
+ EnvFactory.getEnv().getTableSessionConnectionWithDB(DATABASE_NAME)) {
+ try {
+ session.executeQueryStatement(
+ "copy table1(time,tag1,tag2,s1) to 'dup.tsfile' with
(tags(tag1,tag1), memory_threshold 1000000)");
+ Assert.fail("Should report duplicate tag column error");
+ } catch (StatementExecutionException e) {
+ Assert.assertTrue(
+ e.getMessage(), e.getMessage().contains("Duplicate tag column in
TAGS clause: tag1"));
+ }
+ }
+ }
+
+ @Test
+ public void testDuplicateOption()
+ throws IoTDBConnectionException, StatementExecutionException,
IOException {
+ try (ITableSession session =
+ EnvFactory.getEnv().getTableSessionConnectionWithDB(DATABASE_NAME)) {
+ try {
+ session.executeQueryStatement(
+ "copy table1(time,tag1,tag2,s1) to 'dup_option.tsfile' with
(tags(tag1), tags(tag2), memory_threshold 1000000)");
+ Assert.fail("Should report duplicate option error");
+ } catch (StatementExecutionException e) {
+ Assert.assertTrue(
+ e.getMessage(), e.getMessage().contains("Duplicate option in COPY
TO statement: TAGS"));
+ }
+ }
+ }
+
@Test
public void testCopyWithSpecifiedTag()
throws IoTDBConnectionException, StatementExecutionException,
IOException {
@@ -474,7 +506,7 @@ public class IoTDBCopyToTsFileIT {
Assert.assertEquals(2, rowCount);
Assert.assertEquals(1, deviceCount);
Assert.assertTrue(sizeInBytes > 0);
- Assert.assertEquals("default", tableName);
+ Assert.assertEquals("default(auto_gen)", tableName);
Assert.assertEquals("time(auto_gen)", timeColumn);
Assert.assertEquals("[]", tagColumns);
@@ -515,7 +547,7 @@ public class IoTDBCopyToTsFileIT {
Assert.assertEquals(2, rowCount);
Assert.assertEquals(1, deviceCount);
Assert.assertTrue(sizeInBytes > 0);
- Assert.assertEquals("default", tableName);
+ Assert.assertEquals("default(auto_gen)", tableName);
Assert.assertEquals("time(auto_gen)", timeColumn);
Assert.assertEquals("[]", tagColumns);
@@ -648,7 +680,7 @@ public class IoTDBCopyToTsFileIT {
Assert.assertEquals(1, rowCount);
Assert.assertEquals(1, deviceCount);
Assert.assertTrue(sizeInBytes > 0);
- Assert.assertEquals("default", tableName);
+ Assert.assertEquals("default(auto_gen)", tableName);
Assert.assertEquals("time", timeColumn);
Assert.assertEquals("[]", tagColumns);
diff --git
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index 40e7ed020c3..0e9e8158ac7 100644
---
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -2424,6 +2424,10 @@ public final class DataNodeQueryMessages {
"Note that the syntax for 'set configuration' in the tree model is not
exactly the same as that in the table model.";
public static final String UNSUPPORTED_COPY_TO_FORMAT_S_SUPPORTED_FORMATS_S =
"Unsupported COPY TO format '%s'. Supported formats: %s";
+ public static final String
EXCEPTION_DUPLICATE_TAG_COLUMN_IN_TAGS_CLAUSE_ARG_61FD5422 =
+ "Duplicate tag column in TAGS clause: %s";
+ public static final String
EXCEPTION_DUPLICATE_OPTION_IN_COPY_TO_STATEMENT_ARG_99CFE09F =
+ "Duplicate option in COPY TO statement: %s";
public static final String
SIMULTANEOUS_SETTING_OF_MONTHLY_AND_NON_MONTHLY_INTERVALS_IS_NOT_SUPPORTED =
"Simultaneous setting of monthly and non-monthly intervals is not
supported.";
public static final String
DON_T_NEED_TO_SPECIFY_TIME_COLUMN_WHILE_EITHER_TIME_BOUND_OR_FILL_GROUP_PARAMETER_IS_NOT
=
diff --git
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index f236a57f6a4..1ab922fa121 100644
---
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -2799,6 +2799,10 @@ public final class DataNodeQueryMessages {
public static final String UNSUPPORTED_COPY_TO_FORMAT_S_SUPPORTED_FORMATS_S =
"不支持 COPY_TO 格式 '%s'。支持的格式：%s";
+ public static final String
EXCEPTION_DUPLICATE_TAG_COLUMN_IN_TAGS_CLAUSE_ARG_61FD5422 =
+ "TAGS 子句中存在重复的 TAG 列：%s";
+ public static final String
EXCEPTION_DUPLICATE_OPTION_IN_COPY_TO_STATEMENT_ARG_99CFE09F =
+ "COPY TO 语句中存在重复的选项：%s";
public static final String
SIMULTANEOUS_SETTING_OF_MONTHLY_AND_NON_MONTHLY_INTERVALS_IS_NOT_SUPPORTED =
"不支持同时设置月级和非月级时间间隔。";
public static final String
DON_T_NEED_TO_SPECIFY_TIME_COLUMN_WHILE_EITHER_TIME_BOUND_OR_FILL_GROUP_PARAMETER_IS_NOT
=
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/tsfile/CopyToTsFileOptions.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/tsfile/CopyToTsFileOptions.java
index 17f2b689356..22b2e25a6df 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/tsfile/CopyToTsFileOptions.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/tsfile/CopyToTsFileOptions.java
@@ -62,6 +62,7 @@ public class CopyToTsFileOptions implements CopyToOptions {
private final long targetMemoryThreshold;
private boolean generateNewTimeColumn = false;
+ private boolean generateNewTableName = false;
public CopyToTsFileOptions(
String targetTableName,
@@ -82,6 +83,10 @@ public class CopyToTsFileOptions implements CopyToOptions {
return generateNewTimeColumn;
}
+ public boolean isGenerateNewTableName() {
+ return generateNewTableName;
+ }
+
@Override
public void infer(
Analysis analysis, RelationPlan queryRelationPlan, List<ColumnHeader>
columnHeaders) {
@@ -91,8 +96,12 @@ public class CopyToTsFileOptions implements CopyToOptions {
onlyOneQueriedTable = tables.iterator().next();
}
if (targetTableName == null) {
- targetTableName =
- onlyOneQueriedTable == null ? DEFAULT_TABLE_NAME :
onlyOneQueriedTable.getTableName();
+ if (onlyOneQueriedTable == null) {
+ targetTableName = DEFAULT_TABLE_NAME;
+ generateNewTableName = true;
+ } else {
+ targetTableName = onlyOneQueriedTable.getTableName();
+ }
}
if (onlyOneQueriedTable != null) {
inferTimeAndTagsWithTable(onlyOneQueriedTable, columnHeaders);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/tsfile/TsFileFormatCopyToWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/tsfile/TsFileFormatCopyToWriter.java
index 65a8b239db7..08b8ce127ba 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/tsfile/TsFileFormatCopyToWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/tsfile/TsFileFormatCopyToWriter.java
@@ -52,6 +52,7 @@ public class TsFileFormatCopyToWriter implements
IFormatCopyToWriter {
private final String targetTimeColumn;
private final Set<String> targetTagColumns;
private final boolean generateNewTimeColumn;
+ private final boolean generateNewTableName;
private MemoizedCheckedSupplier<TableTsBlock2TsFileWriter, IOException>
tsFileWriter;
private long rowCount = 0;
@@ -66,6 +67,7 @@ public class TsFileFormatCopyToWriter implements
IFormatCopyToWriter {
this.targetTableName = copyToOptions.getTargetTableName();
this.targetTimeColumn = copyToOptions.getTargetTimeColumn();
this.generateNewTimeColumn = copyToOptions.isGenerateNewTimeColumn();
+ this.generateNewTableName = copyToOptions.isGenerateNewTableName();
targetTagColumns = copyToOptions.getTargetTagColumns();
List<ColumnSchema> columnSchemas =
@@ -165,7 +167,9 @@ public class TsFileFormatCopyToWriter implements
IFormatCopyToWriter {
builder.getValueColumnBuilders()[2].writeLong(deviceCount);
builder.getValueColumnBuilders()[3].writeLong(targetFile.length());
builder.getValueColumnBuilders()[4].writeBinary(
- new Binary(targetTableName, TSFileConfig.STRING_CHARSET));
+ new Binary(
+ targetTableName + (generateNewTableName ? AUTO_GEN_MARK : ""),
+ TSFileConfig.STRING_CHARSET));
builder.getValueColumnBuilders()[5].writeBinary(
new Binary(
targetTimeColumn + (generateNewTimeColumn ? AUTO_GEN_MARK : ""),
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
index 7db81826ab6..87e1b417e44 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
@@ -2270,8 +2270,17 @@ public class AstBuilder extends
RelationalSqlBaseVisitor<Node> {
String targetFileName = parseStringLiteral(ctx.fileName.getText());
CopyToOptions.Builder copyToOptionsBuilder = new CopyToOptions.Builder();
if (optionsContext != null) {
+ Set<String> optionKeys = new HashSet<>();
for (RelationalSqlParser.CopyToStatementOptionContext context :
optionsContext.copyToStatementOption()) {
+ String optionKey = getCopyToOptionKey(context);
+ if (!optionKeys.add(optionKey)) {
+ throw new SemanticException(
+ String.format(
+ DataNodeQueryMessages
+
.EXCEPTION_DUPLICATE_OPTION_IN_COPY_TO_STATEMENT_ARG_99CFE09F,
+ optionKey));
+ }
addCopyToOption(copyToOptionsBuilder, context);
}
}
@@ -2296,6 +2305,20 @@ public class AstBuilder extends
RelationalSqlBaseVisitor<Node> {
return new CopyTo(queryNode, targetFileName, copyToOptionsBuilder.build());
}
+ private String
getCopyToOptionKey(RelationalSqlParser.CopyToStatementOptionContext context) {
+ if (context.FORMAT() != null) {
+ return "FORMAT";
+ } else if (context.TABLE() != null) {
+ return "TABLE";
+ } else if (context.TIME() != null) {
+ return "TIME";
+ } else if (context.TAGS() != null) {
+ return "TAGS";
+ } else {
+ return "MEMORY_THRESHOLD";
+ }
+ }
+
private void addCopyToOption(
CopyToOptions.Builder builder,
RelationalSqlParser.CopyToStatementOptionContext context) {
if (context.FORMAT() != null) {
@@ -2325,7 +2348,13 @@ public class AstBuilder extends
RelationalSqlBaseVisitor<Node> {
context.identifierList().identifier();
Set<String> targetTagColumns = new
LinkedHashSet<>(identifierList.size());
for (RelationalSqlParser.IdentifierContext identifierContext :
identifierList) {
- targetTagColumns.add(((Identifier)
visit(identifierContext)).getValue());
+ String tagColumnName = ((Identifier)
visit(identifierContext)).getValue();
+ if (!targetTagColumns.add(tagColumnName)) {
+ throw new SemanticException(
+ String.format(
+
DataNodeQueryMessages.EXCEPTION_DUPLICATE_TAG_COLUMN_IN_TAGS_CLAUSE_ARG_61FD5422,
+ tagColumnName));
+ }
}
builder.withTargetTagColumns(targetTagColumns);
} else if (context.MEMORY_THRESHOLD() != null) {
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java
index 18653a52c57..87c6a01097b 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java
@@ -840,7 +840,7 @@ public class ColumnHeaderConstant {
public static final List<ColumnHeader> COPY_TO_TSFILE_COLUMN_HEADERS =
ImmutableList.of(
- new ColumnHeader(PATH.toLowerCase(), TSDataType.TEXT),
+ new ColumnHeader(PATH.toLowerCase(), TSDataType.STRING),
new ColumnHeader(ROW_COUNT, TSDataType.INT64),
new ColumnHeader(DEVICE_COUNT, TSDataType.INT64),
new ColumnHeader(SIZE_IN_BYTES_TABLE_MODEL, TSDataType.INT64),
