This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 49a7739 [SPARK-51793] Support `ddlParse` and `jsonToDdl` in
`SparkConnectClient`
49a7739 is described below
commit 49a7739a486d855647b0ee145152c027313640fa
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Tue Apr 15 09:12:02 2025 +0900
[SPARK-51793] Support `ddlParse` and `jsonToDdl` in `SparkConnectClient`
### What changes were proposed in this pull request?
This PR aims to support `ddlParse` and `jsonToDdl` in `SparkConnectClient`.
### Why are the changes needed?
For feature parity.
### Does this PR introduce _any_ user-facing change?
No, these will be used internally.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #57 from dongjoon-hyun/SPARK-51793.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/SparkConnectClient.swift | 42 ++++++++++++++++++++++
.../SparkConnectClientTests.swift | 20 +++++++++++
2 files changed, 62 insertions(+)
diff --git a/Sources/SparkConnect/SparkConnectClient.swift
b/Sources/SparkConnect/SparkConnectClient.swift
index d11c4d9..1e8087c 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -438,4 +438,46 @@ public actor SparkConnectClient {
public func clearTags() {
tags.removeAll()
}
+
+ /// Parse a DDL string to ``Spark_Connect_DataType`` instance.
+ /// - Parameter ddlString: A string to parse.
+ /// - Returns: A ``Spark_Connect_DataType`` instance.
+ func ddlParse(_ ddlString: String) async throws -> Spark_Connect_DataType {
+ try await withGRPCClient(
+ transport: .http2NIOPosix(
+ target: .dns(host: self.host, port: self.port),
+ transportSecurity: .plaintext
+ )
+ ) { client in
+ let service = SparkConnectService.Client(wrapping: client)
+ let request = analyze(self.sessionID!, {
+ var ddlParse = AnalyzePlanRequest.DDLParse()
+ ddlParse.ddlString = ddlString
+ return OneOf_Analyze.ddlParse(ddlParse)
+ })
+ let response = try await service.analyzePlan(request)
+ return response.ddlParse.parsed
+ }
+ }
+
+ /// Convert an JSON string to a DDL string.
+ /// - Parameter jsonString: A JSON string.
+ /// - Returns: A DDL string.
+ func jsonToDdl(_ jsonString: String) async throws -> String {
+ try await withGRPCClient(
+ transport: .http2NIOPosix(
+ target: .dns(host: self.host, port: self.port),
+ transportSecurity: .plaintext
+ )
+ ) { client in
+ let service = SparkConnectService.Client(wrapping: client)
+ let request = analyze(self.sessionID!, {
+ var jsonToDDL = AnalyzePlanRequest.JsonToDDL()
+ jsonToDDL.jsonString = jsonString
+ return OneOf_Analyze.jsonToDdl(jsonToDDL)
+ })
+ let response = try await service.analyzePlan(request)
+ return response.jsonToDdl.ddlString
+ }
+ }
}
diff --git a/Tests/SparkConnectTests/SparkConnectClientTests.swift
b/Tests/SparkConnectTests/SparkConnectClientTests.swift
index 965fb53..13dbf4d 100644
--- a/Tests/SparkConnectTests/SparkConnectClientTests.swift
+++ b/Tests/SparkConnectTests/SparkConnectClientTests.swift
@@ -62,4 +62,24 @@ struct SparkConnectClientTests {
#expect(await client.getExecutePlanRequest(plan).tags.isEmpty)
await client.stop()
}
+
+ @Test
+ func ddlParse() async throws {
+ let client = SparkConnectClient(remote: "sc://localhost", user: "test")
+ let _ = try await client.connect(UUID().uuidString)
+ #expect(try await client.ddlParse("a int").simpleString == "struct<a:int>")
+ await client.stop()
+ }
+
+#if !os(Linux) // TODO: Enable this with the offical Spark 4 docker image
+ @Test
+ func jsonToDdl() async throws {
+ let client = SparkConnectClient(remote: "sc://localhost", user: "test")
+ let _ = try await client.connect(UUID().uuidString)
+ let json =
+
#"{"type":"struct","fields":[{"name":"id","type":"long","nullable":false,"metadata":{}}]}"#
+ #expect(try await client.jsonToDdl(json) == "id BIGINT NOT NULL")
+ await client.stop()
+ }
+#endif
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
