This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 28b2d4eb85 [Fix](Export) check properties for export (#21768)
28b2d4eb85 is described below
commit 28b2d4eb85f6743875200adaf36a0f6bed66f068
Author: Tiewei Fang <[email protected]>
AuthorDate: Fri Jul 14 00:15:12 2023 +0800
[Fix](Export) check properties for export (#21768)
---
.../Manipulation/EXPORT.md | 1 +
.../Manipulation/EXPORT.md | 1 +
.../java/org/apache/doris/analysis/ExportStmt.java | 22 +++++++++++++++++++++-
.../java/org/apache/doris/common/FeConstants.java | 6 +++---
.../main/java/org/apache/doris/load/ExportJob.java | 19 +++++++++++++------
5 files changed, 39 insertions(+), 10 deletions(-)
diff --git
a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/EXPORT.md
b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/EXPORT.md
index 7610722e6f..3ebf9e6f93 100644
---
a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/EXPORT.md
+++
b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/EXPORT.md
@@ -74,6 +74,7 @@ illustrate:
- `exec_mem_limit`: Export the upper limit of the memory usage of a single
BE node, the default is 2GB, and the unit is bytes.
- `timeout`: The timeout period of the export job, the default is 2 hours,
the unit is seconds.
- `tablet_num_per_task`: The maximum number of tablets each subtask can
allocate to scan.
+ - `format`: Specifies the file format, currently only supports csv,
csv_with_names, csv_with_names_and_types. If without specified, the default is
csv.
- `WITH BROKER`
diff --git
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/EXPORT.md
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/EXPORT.md
index a64e0034ee..9e79d854a6 100644
---
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/EXPORT.md
+++
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/EXPORT.md
@@ -74,6 +74,7 @@ WITH BROKER
- `exec_mem_limit`：导出在单个 BE 节点的内存使用上限，默认为 2GB，单位为字节。
- `timeout`：导出作业的超时时间，默认为2小时，单位是秒。
- `tablet_num_per_task`：每个子任务能分配扫描的最大 Tablet 数量。
+ -
`format`:指定导出的格式，当前只支持csv、csv_with_names、csv_with_names_and_types。在不指定的情况下，默认为csv。
- `WITH BROKER`
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
index eb35fe4c0d..132cfa3f77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
@@ -39,6 +39,7 @@ import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -63,6 +64,18 @@ public class ExportStmt extends StatementBase {
private static final String DEFAULT_COLUMN_SEPARATOR = "\t";
private static final String DEFAULT_LINE_DELIMITER = "\n";
private static final String DEFAULT_COLUMNS = "";
+
+ private static final ImmutableSet<String> PROPERTIES_SET = new
ImmutableSet.Builder<String>()
+ .add(ExportStmt.LABEL)
+ .add(LoadStmt.EXEC_MEM_LIMIT)
+ .add(LoadStmt.TIMEOUT_PROPERTY)
+ .add(ExportStmt.TABLET_NUMBER_PER_TASK_PROP)
+ .add(LoadStmt.KEY_IN_PARAM_COLUMNS)
+ .add("line_delimiter")
+ .add("column_separator")
+ .add("format")
+ .build();
+
private TableName tblName;
private List<String> partitions;
private Expr whereExpr;
@@ -271,7 +284,6 @@ public class ExportStmt extends StatementBase {
properties, ExportStmt.DEFAULT_COLUMN_SEPARATOR));
this.lineDelimiter =
Separator.convertSeparator(PropertyAnalyzer.analyzeLineDelimiter(
properties, ExportStmt.DEFAULT_LINE_DELIMITER));
- this.columns = properties.get(LoadStmt.KEY_IN_PARAM_COLUMNS);
// exec_mem_limit
if (properties.containsKey(LoadStmt.EXEC_MEM_LIMIT)) {
try {
@@ -315,6 +327,14 @@ public class ExportStmt extends StatementBase {
String label = "export_" + UUID.randomUUID().toString();
properties.put(LABEL, label);
}
+
+ for (String key : properties.keySet()) {
+ if (!PROPERTIES_SET.contains(key)) {
+ throw new DdlException("Invalid property key: '" + key + "'");
+ }
+ }
+
+ this.columns = properties.get(LoadStmt.KEY_IN_PARAM_COLUMNS);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
index 91ad6e4a61..3c3ae2864b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
@@ -63,9 +63,9 @@ public class FeConstants {
public static String null_string = "\\N";
public static long tablet_checker_interval_ms = 20 * 1000L;
- public static String csv = "csv";
- public static String csv_with_names = "csv_with_names";
- public static String csv_with_names_and_types = "csv_with_names_and_types";
+ public static final String csv = "csv";
+ public static final String csv_with_names = "csv_with_names";
+ public static final String csv_with_names_and_types =
"csv_with_names_and_types";
public static String text = "text";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index 36bdaa993c..cbb6f403aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -285,15 +285,22 @@ public class ExportJob implements Writable {
return types;
}
- private String genHeader(Map<String, String> properties) {
+ private String genHeader(Map<String, String> properties) throws
UserException {
String header = "";
if (properties.containsKey("format")) {
String headerType = properties.get("format");
- if (headerType.equals(FeConstants.csv_with_names)) {
- header = genNames();
- } else if
(headerType.equals(FeConstants.csv_with_names_and_types)) {
- header = genNames();
- header += genTypes();
+ switch (headerType) {
+ case FeConstants.csv:
+ break;
+ case FeConstants.csv_with_names:
+ header = genNames();
+ break;
+ case FeConstants.csv_with_names_and_types:
+ header = genNames();
+ header += genTypes();
+ break;
+ default:
+ throw new DdlException("Unknown format for export: " +
headerType);
}
}
return header;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
