This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new c3ad3c3 [feature] support read json type (#174)
c3ad3c3 is described below
commit c3ad3c3df7de58dd3a21e7baf84c1a94f022fad8
Author: gnehil <[email protected]>
AuthorDate: Tue Dec 26 11:02:36 2023 +0800
[feature] support read json type (#174)
---
.gitignore | 29 ++++++++++++++++++++++
.../org/apache/doris/spark/sql/SchemaUtils.scala | 3 ++-
.../apache/doris/spark/sql/TestSchemaUtils.scala | 3 +++
3 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/.gitignore b/.gitignore
index 81040cb..4413142 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,32 @@ spark-doris-connector/dependency-reduced-pom.xml
spark-doris-connector/output/
spark-doris-connector/target/
spark-doris-connector/.idea/
+
+
+### Java template
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see
http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+replay_pid*
+
+*.iml
diff --git
a/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/SchemaUtils.scala
b/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/SchemaUtils.scala
index 032f16b..e298349 100644
---
a/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/SchemaUtils.scala
+++
b/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/SchemaUtils.scala
@@ -118,6 +118,7 @@ private[spark] object SchemaUtils {
case "CHAR" => DataTypes.StringType
case "LARGEINT" => DecimalType(38,0)
case "VARCHAR" => DataTypes.StringType
+ case "JSON" => DataTypes.StringType
case "JSONB" => DataTypes.StringType
case "DECIMALV2" => DecimalType(precision, scale)
case "DECIMAL32" => DecimalType(precision, scale)
@@ -130,7 +131,7 @@ private[spark] object SchemaUtils {
case "STRUCT" => DataTypes.StringType
case "HLL" =>
throw new DorisException("Unsupported type " + dorisType)
- case _ =>
+ case _ =>
throw new DorisException("Unrecognized Doris type " + dorisType)
}
}
diff --git
a/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestSchemaUtils.scala
b/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestSchemaUtils.scala
index 98ec67f..da797e2 100644
---
a/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestSchemaUtils.scala
+++
b/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestSchemaUtils.scala
@@ -57,6 +57,7 @@ class TestSchemaUtils extends ExpectedExceptionTest {
Assert.assertEquals(DataTypes.DoubleType,
SchemaUtils.getCatalystType("DOUBLE", 0, 0))
Assert.assertEquals(DataTypes.DateType,
SchemaUtils.getCatalystType("DATE", 0, 0))
Assert.assertEquals(DataTypes.StringType,
SchemaUtils.getCatalystType("DATETIME", 0, 0))
+ Assert.assertEquals(DataTypes.StringType,
SchemaUtils.getCatalystType("DATETIMEV2", 0, 0))
Assert.assertEquals(DataTypes.BinaryType,
SchemaUtils.getCatalystType("BINARY", 0, 0))
Assert.assertEquals(DecimalType(9, 3),
SchemaUtils.getCatalystType("DECIMAL", 9, 3))
Assert.assertEquals(DataTypes.StringType,
SchemaUtils.getCatalystType("CHAR", 0, 0))
@@ -65,6 +66,8 @@ class TestSchemaUtils extends ExpectedExceptionTest {
Assert.assertEquals(DecimalType(10, 5),
SchemaUtils.getCatalystType("DECIMALV2", 10, 5))
Assert.assertEquals(DataTypes.DoubleType,
SchemaUtils.getCatalystType("TIME", 0, 0))
Assert.assertEquals(DataTypes.StringType,
SchemaUtils.getCatalystType("STRING", 0, 0))
+ Assert.assertEquals(DataTypes.StringType,
SchemaUtils.getCatalystType("JSON", 0, 0))
+ Assert.assertEquals(DataTypes.StringType,
SchemaUtils.getCatalystType("JSONB", 0, 0))
thrown.expect(classOf[DorisException])
thrown.expectMessage(startsWith("Unsupported type"))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
