This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 1fbe1b6 [SPARK-57049] Support `listPartitions` in `Catalog`
1fbe1b6 is described below
commit 1fbe1b658d9d11190cfd098c186e3a767d54b448
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun May 24 21:40:53 2026 -0700
[SPARK-57049] Support `listPartitions` in `Catalog`
### What changes were proposed in this pull request?
This PR aims to support `Spark_Connect_ListPartitions` added in Apache
Spark Connect 4.2.0-preview5
- https://github.com/apache/spark/pull/55025
### Why are the changes needed?
To improve API coverage.
### Does this PR introduce _any_ user-facing change?
No because this is a new API addition.
### How was this patch tested?
Pass the CIs with a newly added test case.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.7
Closes #384 from dongjoon-hyun/SPARK-57049.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/Catalog.swift | 21 +++++++++++++++++++++
Tests/SparkConnectTests/CatalogTests.swift | 20 ++++++++++++++++++++
2 files changed, 41 insertions(+)
diff --git a/Sources/SparkConnect/Catalog.swift
b/Sources/SparkConnect/Catalog.swift
index fea05fa..7e2b45a 100644
--- a/Sources/SparkConnect/Catalog.swift
+++ b/Sources/SparkConnect/Catalog.swift
@@ -74,6 +74,10 @@ public struct Function: Sendable, Equatable {
public var isTemporary: Bool
}
+public struct TablePartition: Sendable, Equatable {
+ public var partition: String
+}
+
/// Interface through which the user may create, drop, alter or query
underlying databases, tables, functions etc.
/// To access this, use ``SparkSession.catalog``.
public actor Catalog: Sendable {
@@ -430,6 +434,23 @@ public actor Catalog: Sendable {
return df
}
+ /// Returns a list of partitions for the given table.
+ /// - Parameter tableName: a qualified or unqualified name that designates a
table. If no database
+ /// identifier is provided, it refers to a table in the current database.
+ /// - Returns: A list of ``TablePartition``.
+ public func listPartitions(_ tableName: String) async throws ->
[TablePartition] {
+ let df = getDataFrame({
+ var listPartitions = Spark_Connect_ListPartitions()
+ listPartitions.tableName = tableName
+ var catalog = Spark_Connect_Catalog()
+ catalog.listPartitions = listPartitions
+ return catalog
+ })
+ return try await df.collect().map {
+ try TablePartition(partition: $0[0] as! String)
+ }
+ }
+
/// Check if the function with the specified name exists. This can either be
a temporary function
/// or a function.
/// - Parameter functionName: a qualified or unqualified name that
designates a function. It follows the same
diff --git a/Tests/SparkConnectTests/CatalogTests.swift
b/Tests/SparkConnectTests/CatalogTests.swift
index ebd7ae0..5a4e70f 100644
--- a/Tests/SparkConnectTests/CatalogTests.swift
+++ b/Tests/SparkConnectTests/CatalogTests.swift
@@ -308,6 +308,26 @@ struct CatalogTests {
await spark.stop()
}
+ @Test
+ func listPartitions() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ if await spark.version >= "4.2" {
+ let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of:
"-", with: "")
+ try await SQLHelper.withTable(spark, tableName)({
+ try await spark.sql(
+ "CREATE TABLE \(tableName) (id INT, p INT) USING parquet PARTITIONED
BY (p)"
+ ).count()
+ #expect(try await spark.catalog.listPartitions(tableName).isEmpty)
+
+ try await spark.sql("INSERT INTO \(tableName) VALUES (1, 1), (2,
2)").count()
+ let partitions = try await spark.catalog.listPartitions(tableName)
+ #expect(partitions.count == 2)
+ #expect(Set(partitions.map { $0.partition }) == Set(["p=1", "p=2"]))
+ })
+ }
+ await spark.stop()
+ }
+
@Test
func functionExists() async throws {
let spark = try await SparkSession.builder.getOrCreate()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
