This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-c108335-hive-sql
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-c108335-hive-sql by
this push:
new 808ef61620c [tmp] support skip.header.line.count
808ef61620c is described below
commit 808ef61620c75f5726be62daea731bdd0ff736cf
Author: morningman <[email protected]>
AuthorDate: Tue Apr 8 06:13:08 2025 -0700
[tmp] support skip.header.line.count
---
.../java/org/apache/doris/datasource/hive/HiveProperties.java | 9 +++++++++
.../org/apache/doris/datasource/hive/source/HiveScanNode.java | 2 ++
2 files changed, 11 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java
index 74f3dcc1a9d..b098b9d43b7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.hive;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.serde2.OpenCSVSerde;
@@ -131,6 +132,14 @@ public class HiveProperties {
return
HiveMetaStoreClientHelper.firstPresentOrDefault(DEFAULT_ESCAPE_CHAR,
escapeChar);
}
+ public static int getCsvSkipHeaderLine(Table table) {
+ String valueFromTbl =
table.getParameters().get("skip.header.line.count");
+ if (!Strings.isNullOrEmpty(valueFromTbl)) {
+ return Integer.valueOf(valueFromTbl);
+ }
+ return 0;
+ }
+
// Set properties to table
public static void setTableProperties(Table table, Map<String, String>
properties) {
HashMap<String, String> serdeProps = new HashMap<>();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index 890f6147f33..7255a54de98 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -431,6 +431,8 @@ public class HiveScanNode extends FileQueryScanNode {
Table table = hmsTable.getRemoteTable();
// TODO: separate hive text table and OpenCsv table
String serDeLib = table.getSd().getSerdeInfo().getSerializationLib();
+ int skipHeaderLine = HiveProperties.getCsvSkipHeaderLine(table);
+ fileAttributes.setSkipLines(skipHeaderLine);
if
(serDeLib.equals("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")) {
TFileTextScanRangeParams textParams = new
TFileTextScanRangeParams();
// set properties of LazySimpleSerDe
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
