This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch rc/1.3.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rc/1.3.3 by this push:
new 3a5304a78da Load: New SQL grammer -> LOAD <filePath> with ('k'='v',
...) (#13553)
3a5304a78da is described below
commit 3a5304a78daba9f47e5bbecbc957554b45ba00ed
Author: YC27 <[email protected]>
AuthorDate: Fri Sep 20 13:21:01 2024 +0800
Load: New SQL grammer -> LOAD <filePath> with ('k'='v', ...) (#13553)
Co-authored-by: Steve Yurong Su <[email protected]>
(cherry picked from commit 3f922e72c59ea869868067ac52912fc0e80bb718)
---
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 8 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 13 ++-
.../db/queryengine/plan/parser/ASTVisitor.java | 22 ++++-
.../plan/statement/crud/LoadTsFileStatement.java | 18 ++++
.../load/config/LoadTsFileConfigurator.java | 100 +++++++++++++++++++++
5 files changed, 157 insertions(+), 4 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
index ca5aca30b8d..11892f32921 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
@@ -542,7 +542,9 @@ public class IoTDBLoadTsFileIT {
final Statement statement = connection.createStatement()) {
statement.execute(
- String.format("load \"%s\" sglevel=2 onSuccess=none",
file1.getAbsolutePath()));
+ String.format(
+ "load \"%s\" with ('database-level'='2', 'on-success'='none')",
+ file1.getAbsolutePath()));
try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by
level=1,2")) {
@@ -560,7 +562,9 @@ public class IoTDBLoadTsFileIT {
final Statement statement = connection.createStatement()) {
statement.execute(
- String.format("load \"%s\" sglevel=2 onSuccess=delete",
file2.getAbsolutePath()));
+ String.format(
+ "load \"%s\" with ('database-level'='2', 'on-success'='delete')",
+ file2.getAbsolutePath()));
try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by
level=1,2")) {
diff --git
a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 1d45ccb3d36..d742d79ac8d 100644
---
a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++
b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -1128,7 +1128,7 @@ loadTimeseries
// Load TsFile
loadFile
- : LOAD fileName=STRING_LITERAL loadFileAttributeClauses?
+ : LOAD fileName=STRING_LITERAL ((loadFileAttributeClauses?) |
(loadFileWithAttributeClauses))
;
loadFileAttributeClauses
@@ -1141,6 +1141,17 @@ loadFileAttributeClause
| ONSUCCESS operator_eq (DELETE|NONE)
;
+loadFileWithAttributeClauses
+ : WITH
+ LR_BRACKET
+ (loadFileWithAttributeClause COMMA)* loadFileWithAttributeClause?
+ RR_BRACKET
+ ;
+
+loadFileWithAttributeClause
+ : loadFileWithKey=STRING_LITERAL OPERATOR_SEQ
loadFileWithValue=STRING_LITERAL
+ ;
+
// Remove TsFile
removeFile
: REMOVE fileName=STRING_LITERAL
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index 099678ffcee..af3453c1e73 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -216,6 +216,7 @@ import
org.apache.iotdb.db.queryengine.plan.statement.sys.quota.SetThrottleQuota
import
org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowSpaceQuotaStatement;
import
org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuotaStatement;
import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType;
+import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator;
import org.apache.iotdb.db.utils.DateTimeUtils;
import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.db.utils.constant.SqlConstant;
@@ -2003,8 +2004,27 @@ public class ASTVisitor extends
IoTDBSqlParserBaseVisitor<Statement> {
@Override
public Statement visitLoadFile(IoTDBSqlParser.LoadFileContext ctx) {
try {
- LoadTsFileStatement loadTsFileStatement =
+ final LoadTsFileStatement loadTsFileStatement =
new LoadTsFileStatement(parseStringLiteral(ctx.fileName.getText()));
+
+ // if sql have with, return new load sql statement
+ if (ctx.loadFileWithAttributeClauses() != null) {
+ final Map<String, String> loadTsFileAttributes = new HashMap<>();
+ for (IoTDBSqlParser.LoadFileWithAttributeClauseContext
attributeContext :
+ ctx.loadFileWithAttributeClauses().loadFileWithAttributeClause()) {
+ final String key =
+
parseStringLiteral(attributeContext.loadFileWithKey.getText()).trim().toLowerCase();
+ final String value =
+
parseStringLiteral(attributeContext.loadFileWithValue.getText()).trim().toLowerCase();
+
+ LoadTsFileConfigurator.validateParameters(key, value);
+ loadTsFileAttributes.put(key, value);
+ }
+
+ loadTsFileStatement.setLoadAttributes(loadTsFileAttributes);
+ return loadTsFileStatement;
+ }
+
if (ctx.loadFileAttributeClauses() != null) {
for (IoTDBSqlParser.LoadFileAttributeClauseContext attributeContext :
ctx.loadFileAttributeClauses().loadFileAttributeClause()) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
index 06fa870aee6..1fb3e5f1b92 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
@@ -26,6 +26,7 @@ import
org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.common.constant.TsFileConstant;
@@ -36,6 +37,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
public class LoadTsFileStatement extends Statement {
@@ -45,6 +47,8 @@ public class LoadTsFileStatement extends Statement {
private boolean deleteAfterLoad;
private boolean autoCreateDatabase;
+ private Map<String, String> loadAttributes;
+
private final List<File> tsFiles;
private final List<TsFileResource> resources;
private final List<Long> writePointCountList;
@@ -60,6 +64,10 @@ public class LoadTsFileStatement extends Statement {
this.writePointCountList = new ArrayList<>();
this.statementType = StatementType.MULTI_BATCH_INSERT;
+ processTsFile(filePath);
+ }
+
+ private void processTsFile(final String filePath) throws
FileNotFoundException {
if (file.isFile()) {
tsFiles.add(file);
} else {
@@ -165,6 +173,16 @@ public class LoadTsFileStatement extends Statement {
return writePointCountList.get(resourceIndex);
}
+ public void setLoadAttributes(final Map<String, String> loadAttributes) {
+ this.loadAttributes = loadAttributes;
+ initAttributes();
+ }
+
+ private void initAttributes() {
+ this.databaseLevel =
LoadTsFileConfigurator.parseOrGetDefaultDatabaseLevel(loadAttributes);
+ this.deleteAfterLoad =
LoadTsFileConfigurator.parseOrGetDefaultOnSuccess(loadAttributes);
+ }
+
@Override
public List<PartialPath> getPaths() {
return Collections.emptyList();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java
new file mode 100644
index 00000000000..d09540133ab
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.load.config;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.sql.SemanticException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class LoadTsFileConfigurator {
+
+ public static void validateParameters(final String key, final String value) {
+ switch (key) {
+ case DATABASE_LEVEL_KEY:
+ validateDatabaseLevelParam(value);
+ break;
+ case ON_SUCCESS_KEY:
+ validateOnSuccessParam(value);
+ break;
+ default:
+ throw new SemanticException("Invalid parameter '" + key + "' for LOAD
TSFILE command.");
+ }
+ }
+
+ private static final String DATABASE_LEVEL_KEY = "database-level";
+ private static final int DATABASE_LEVEL_DEFAULT_VALUE =
+ IoTDBDescriptor.getInstance().getConfig().getDefaultStorageGroupLevel();
+ private static final int DATABASE_LEVEL_MIN_VALUE = 1;
+
+ public static void validateDatabaseLevelParam(final String databaseLevel) {
+ try {
+ int level = Integer.parseInt(databaseLevel);
+ if (level < DATABASE_LEVEL_MIN_VALUE) {
+ throw new SemanticException(
+ String.format(
+ "Given database level %d is less than the minimum value %d,
please input a valid database level.",
+ level, DATABASE_LEVEL_MIN_VALUE));
+ }
+ } catch (Exception e) {
+ throw new SemanticException(
+ String.format(
+ "Given database level %s is not a valid integer, please input a
valid database level.",
+ databaseLevel));
+ }
+ }
+
+ public static int parseOrGetDefaultDatabaseLevel(final Map<String, String>
loadAttributes) {
+ return Integer.parseInt(
+ loadAttributes.getOrDefault(
+ DATABASE_LEVEL_KEY, String.valueOf(DATABASE_LEVEL_DEFAULT_VALUE)));
+ }
+
+ private static final String ON_SUCCESS_KEY = "on-success";
+ private static final String ON_SUCCESS_DELETE_VALUE = "delete";
+ private static final String ON_SUCCESS_NONE_VALUE = "none";
+ private static final Set<String> ON_SUCCESS_VALUE_SET =
+ Collections.unmodifiableSet(
+ new HashSet<>(Arrays.asList(ON_SUCCESS_DELETE_VALUE,
ON_SUCCESS_NONE_VALUE)));
+
+ public static void validateOnSuccessParam(final String onSuccess) {
+ if (!ON_SUCCESS_VALUE_SET.contains(onSuccess)) {
+ throw new SemanticException(
+ String.format(
+ "Given on-success value '%s' is not supported, please input a
valid on-success value.",
+ onSuccess));
+ }
+ }
+
+ public static boolean parseOrGetDefaultOnSuccess(final Map<String, String>
loadAttributes) {
+ final String value = loadAttributes.get(ON_SUCCESS_KEY);
+ return StringUtils.isEmpty(value) || ON_SUCCESS_DELETE_VALUE.equals(value);
+ }
+
+ private LoadTsFileConfigurator() {
+ throw new IllegalStateException("Utility class");
+ }
+}
