(paimon) branch master updated: [hive] Format table: Support field delimiter in HMS (#5829)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 0904f4d763 [hive] Format table: Support field delimiter in HMS  (#5829)
0904f4d763 is described below

commit 0904f4d7630ef784d95a80bce9af2f0f79e6b12a
Author: Zouxxyy <zouxinyu....@alibaba-inc.com>
AuthorDate: Wed Jul 2 21:32:09 2025 +0800

    [hive] Format table: Support field delimiter in HMS  (#5829)
---
 .../src/main/java/org/apache/paimon/hive/HiveCatalog.java    | 12 +++++++++---
 .../src/main/java/org/apache/paimon/hive/HiveTableUtils.java |  6 ++++--
 .../org/apache/paimon/spark/sql/FormatTableTestBase.scala    |  8 ++++++++
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git 
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
 
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 35747cdb63..85a3d9883e 100644
--- 
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++ 
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -152,6 +152,8 @@ public class HiveCatalog extends AbstractCatalog {
     private static final String HIVE_EXTERNAL_TABLE_PROP = "EXTERNAL";
     private static final int DEFAULT_TABLE_BATCH_SIZE = 300;
     private static final String HIVE_LAST_UPDATE_TIME_PROP = 
"transient_lastDdlTime";
+    // hive default field delimiter is '\u0001'
+    public static final String HIVE_FIELD_DELIM_DEFAULT = "\u0001";
 
     private final HiveConf hiveConf;
     private final String clientClassName;
@@ -1465,10 +1467,14 @@ public class HiveCatalog extends AbstractCatalog {
         return OUTPUT_FORMAT_CLASS_NAME;
     }
 
-    private Map<String, String> setSerDeInfoParam(@Nullable FormatTable.Format 
provider) {
+    private Map<String, String> setSerDeInfoParam(
+            @Nullable FormatTable.Format provider, Map<String, String> 
tableParameters) {
         Map<String, String> param = new HashMap<>();
         if (provider == FormatTable.Format.CSV) {
-            param.put(FIELD_DELIM, options.get(FIELD_DELIMITER));
+            param.put(
+                    FIELD_DELIM,
+                    tableParameters.getOrDefault(
+                            FIELD_DELIMITER.key(), 
options.get(FIELD_DELIMITER)));
         }
         return param;
     }
@@ -1485,7 +1491,7 @@ public class HiveCatalog extends AbstractCatalog {
         sd.setOutputFormat(getOutputFormatClassName(provider));
 
         SerDeInfo serDeInfo = sd.getSerdeInfo() != null ? sd.getSerdeInfo() : 
new SerDeInfo();
-        serDeInfo.setParameters(setSerDeInfoParam(provider));
+        serDeInfo.setParameters(setSerDeInfoParam(provider, schema.options()));
         serDeInfo.setSerializationLib(getSerdeClassName(provider));
         sd.setSerdeInfo(serDeInfo);
 
diff --git 
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java
 
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java
index fc9ec43d4f..fdf520689b 100644
--- 
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java
+++ 
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java
@@ -37,6 +37,7 @@ import static org.apache.paimon.CoreOptions.PATH;
 import static org.apache.paimon.CoreOptions.TYPE;
 import static org.apache.paimon.TableType.FORMAT_TABLE;
 import static org.apache.paimon.catalog.Catalog.COMMENT_PROP;
+import static org.apache.paimon.hive.HiveCatalog.HIVE_FIELD_DELIM_DEFAULT;
 import static org.apache.paimon.hive.HiveCatalog.isView;
 import static org.apache.paimon.table.FormatTableOptions.FIELD_DELIMITER;
 
@@ -76,10 +77,11 @@ class HiveTableUtils {
                 format = Format.JSON;
             } else {
                 format = Format.CSV;
-                // hive default field delimiter is '\u0001'
                 options.set(
                         FIELD_DELIMITER,
-                        serdeInfo.getParameters().getOrDefault(FIELD_DELIM, 
"\u0001"));
+                        serdeInfo
+                                .getParameters()
+                                .getOrDefault(FIELD_DELIM, 
HIVE_FIELD_DELIM_DEFAULT));
             }
         } else {
             throw new UnsupportedOperationException("Unsupported table: " + 
hiveTable);
diff --git 
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FormatTableTestBase.scala
 
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FormatTableTestBase.scala
index 8c5b96a7d6..a296c2ce90 100644
--- 
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FormatTableTestBase.scala
+++ 
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FormatTableTestBase.scala
@@ -117,4 +117,12 @@ abstract class FormatTableTestBase extends 
PaimonHiveTestBase {
       }
     }
   }
+
+  test("Format table: field delimiter in HMS") {
+    withTable("t1") {
+      sql("CREATE TABLE t1 (id INT, p1 INT, p2 INT) USING csv OPTIONS 
('field-delimiter' ';')")
+      val row = sql("SHOW CREATE TABLE t1").collect()(0)
+      assert(row.toString().contains("'field-delimiter' = ';'"))
+    }
+  }
 }