This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new d8c9acfbf3d [SPARK-43932][SQL][PYTHON][CONNECT] Add `current` like
functions to Scala and Python
d8c9acfbf3d is described below
commit d8c9acfbf3dc79e66204bd5c0bac291f1cf34792
Author: Ruifeng Zheng <ruife...@apache.org>
AuthorDate: Wed Jun 14 12:44:33 2023 +0800
[SPARK-43932][SQL][PYTHON][CONNECT] Add `current` like functions to Scala
and Python
### What changes were proposed in this pull request?
Add following functions:
- curdate
- current_catalog
- current_database
- current_schema
- current_timezone
- current_user
- user
to:
- Scala API
- Python API
- Spark Connect Scala Client
- Spark Connect Python Client
### Why are the changes needed?
for function parity
### Does this PR introduce _any_ user-facing change?
yes, new functions
### How was this patch tested?
added ut and doctests
Closes #41516 from zhengruifeng/sql_cur_function.
Authored-by: Ruifeng Zheng <ruife...@apache.org>
Signed-off-by: Ruifeng Zheng <ruife...@apache.org>
---
.../scala/org/apache/spark/sql/functions.scala | 57 +++++++++
.../apache/spark/sql/PlanGenerationTestSuite.scala | 28 ++++
.../explain-results/function_curdate.explain | 2 +
.../function_current_catalog.explain | 2 +
.../function_current_database.explain | 2 +
.../function_current_schema.explain | 2 +
.../function_current_timezone.explain | 2 +
.../explain-results/function_current_user.explain | 2 +
.../explain-results/function_user.explain | 2 +
.../query-tests/queries/function_curdate.json | 20 +++
.../query-tests/queries/function_curdate.proto.bin | Bin 0 -> 114 bytes
.../queries/function_current_catalog.json | 20 +++
.../queries/function_current_catalog.proto.bin | Bin 0 -> 177 bytes
.../queries/function_current_database.json | 20 +++
.../queries/function_current_database.proto.bin | Bin 0 -> 178 bytes
.../queries/function_current_schema.json | 20 +++
.../queries/function_current_schema.proto.bin | Bin 0 -> 176 bytes
.../queries/function_current_timezone.json | 20 +++
.../queries/function_current_timezone.proto.bin | Bin 0 -> 123 bytes
.../query-tests/queries/function_current_user.json | 20 +++
.../queries/function_current_user.proto.bin | Bin 0 -> 174 bytes
.../query-tests/queries/function_user.json | 20 +++
.../query-tests/queries/function_user.proto.bin | Bin 0 -> 166 bytes
.../source/reference/pyspark.sql/functions.rst | 7 +
python/pyspark/sql/connect/functions.py | 49 +++++++
python/pyspark/sql/functions.py | 142 +++++++++++++++++++++
.../scala/org/apache/spark/sql/functions.scala | 57 +++++++++
.../apache/spark/sql/DataFrameFunctionsSuite.scala | 15 +++
.../org/apache/spark/sql/DateFunctionsSuite.scala | 6 +
29 files changed, 515 insertions(+)
diff --git
a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
index 85ff85d7845..7aafbdf1e46 100644
---
a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
+++
b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2835,6 +2835,38 @@ object functions {
// Misc functions
//////////////////////////////////////////////////////////////////////////////////////////////
+ /**
+ * Returns the current catalog.
+ *
+ * @group misc_funcs
+ * @since 3.5.0
+ */
+ def current_catalog(): Column = Column.fn("current_catalog")
+
+ /**
+ * Returns the current database.
+ *
+ * @group misc_funcs
+ * @since 3.5.0
+ */
+ def current_database(): Column = Column.fn("current_database")
+
+ /**
+ * Returns the current schema.
+ *
+ * @group misc_funcs
+ * @since 3.5.0
+ */
+ def current_schema(): Column = Column.fn("current_schema")
+
+ /**
+ * Returns the user name of current execution context.
+ *
+ * @group misc_funcs
+ * @since 3.5.0
+ */
+ def current_user(): Column = Column.fn("current_user")
+
/**
* Calculates the MD5 digest of a binary column and returns the value as a
32 character hex
* string.
@@ -2989,6 +3021,14 @@ object functions {
allowDifferentLgConfigK: Boolean): Column =
hll_union(Column(columnName1), Column(columnName2),
allowDifferentLgConfigK)
+ /**
+ * Returns the user name of current execution context.
+ *
+ * @group misc_funcs
+ * @since 3.5.0
+ */
+ def user(): Column = Column.fn("user")
+
//////////////////////////////////////////////////////////////////////////////////////////////
// String functions
//////////////////////////////////////////////////////////////////////////////////////////////
@@ -3614,6 +3654,15 @@ object functions {
def add_months(startDate: Column, numMonths: Column): Column =
Column.fn("add_months", startDate, numMonths)
+ /**
+ * Returns the current date at the start of query evaluation as a date
column. All calls of
+ * current_date within the same query return the same value.
+ *
+ * @group datetime_funcs
+ * @since 3.5.0
+ */
+ def curdate(): Column = Column.fn("curdate")
+
/**
* Returns the current date at the start of query evaluation as a date
column. All calls of
* current_date within the same query return the same value.
@@ -3623,6 +3672,14 @@ object functions {
*/
def current_date(): Column = Column.fn("current_date")
+ /**
+ * Returns the current session local timezone.
+ *
+ * @group datetime_funcs
+ * @since 3.5.0
+ */
+ def current_timezone(): Column = Column.fn("current_timezone")
+
/**
* Returns the current timestamp at the start of query evaluation as a
timestamp column. All
* calls of current_timestamp within the same query return the same value.
diff --git
a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
index ebb171af9ef..7f41cdb956f 100644
---
a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
+++
b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
@@ -1370,6 +1370,26 @@ class PlanGenerationTestSuite
fn.radians("b")
}
+ functionTest("current_catalog") {
+ fn.current_catalog()
+ }
+
+ functionTest("current_database") {
+ fn.current_database()
+ }
+
+ functionTest("current_schema") {
+ fn.current_schema()
+ }
+
+ functionTest("current_user") {
+ fn.current_user()
+ }
+
+ functionTest("user") {
+ fn.user()
+ }
+
functionTest("md5") {
fn.md5(fn.col("g").cast("binary"))
}
@@ -1659,10 +1679,18 @@ class PlanGenerationTestSuite
fn.add_months(fn.col("d"), 2)
}
+ temporalFunctionTest("curdate") {
+ fn.curdate()
+ }
+
temporalFunctionTest("current_date") {
fn.current_date()
}
+ temporalFunctionTest("current_timezone") {
+ fn.current_timezone()
+ }
+
temporalFunctionTest("current_timestamp") {
fn.current_timestamp()
}
diff --git
a/connector/connect/common/src/test/resources/query-tests/explain-results/function_curdate.explain
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_curdate.explain
new file mode 100644
index 00000000000..5305b346c4f
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_curdate.explain
@@ -0,0 +1,2 @@
+Project [current_date(Some(America/Los_Angeles)) AS current_date()#0]
++- LocalRelation <empty>, [d#0, t#0, s#0, x#0L, wt#0]
diff --git
a/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_catalog.explain
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_catalog.explain
new file mode 100644
index 00000000000..28a55ff2d1f
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_catalog.explain
@@ -0,0 +1,2 @@
+Project [current_catalog() AS current_catalog()#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git
a/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_database.explain
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_database.explain
new file mode 100644
index 00000000000..93dfac524d9
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_database.explain
@@ -0,0 +1,2 @@
+Project [current_database() AS current_database()#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git
a/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_schema.explain
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_schema.explain
new file mode 100644
index 00000000000..93dfac524d9
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_schema.explain
@@ -0,0 +1,2 @@
+Project [current_database() AS current_database()#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git
a/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_timezone.explain
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_timezone.explain
new file mode 100644
index 00000000000..d67b1d59d29
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_timezone.explain
@@ -0,0 +1,2 @@
+Project [current_timezone() AS current_timezone()#0]
++- LocalRelation <empty>, [d#0, t#0, s#0, x#0L, wt#0]
diff --git
a/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_user.explain
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_user.explain
new file mode 100644
index 00000000000..82f5d2adcec
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_current_user.explain
@@ -0,0 +1,2 @@
+Project [current_user() AS current_user()#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git
a/connector/connect/common/src/test/resources/query-tests/explain-results/function_user.explain
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_user.explain
new file mode 100644
index 00000000000..82f5d2adcec
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/explain-results/function_user.explain
@@ -0,0 +1,2 @@
+Project [current_user() AS current_user()#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_curdate.json
b/connector/connect/common/src/test/resources/query-tests/queries/function_curdate.json
new file mode 100644
index 00000000000..c344f527170
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/queries/function_curdate.json
@@ -0,0 +1,20 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema":
"struct\u003cd:date,t:timestamp,s:string,x:bigint,wt:struct\u003cstart:timestamp,end:timestamp\u003e\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "curdate"
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_curdate.proto.bin
b/connector/connect/common/src/test/resources/query-tests/queries/function_curdate.proto.bin
new file mode 100644
index 00000000000..6ec58b57c2a
Binary files /dev/null and
b/connector/connect/common/src/test/resources/query-tests/queries/function_curdate.proto.bin
differ
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_current_catalog.json
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_catalog.json
new file mode 100644
index 00000000000..27c7b231119
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_catalog.json
@@ -0,0 +1,20 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema":
"struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "current_catalog"
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_current_catalog.proto.bin
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_catalog.proto.bin
new file mode 100644
index 00000000000..bb25a499354
Binary files /dev/null and
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_catalog.proto.bin
differ
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_current_database.json
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_database.json
new file mode 100644
index 00000000000..dfa59fd5fe5
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_database.json
@@ -0,0 +1,20 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema":
"struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "current_database"
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_current_database.proto.bin
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_database.proto.bin
new file mode 100644
index 00000000000..fdb11c9c8bd
Binary files /dev/null and
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_database.proto.bin
differ
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_current_schema.json
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_schema.json
new file mode 100644
index 00000000000..01d5126a74b
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_schema.json
@@ -0,0 +1,20 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema":
"struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "current_schema"
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_current_schema.proto.bin
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_schema.proto.bin
new file mode 100644
index 00000000000..9687afe89a5
Binary files /dev/null and
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_schema.proto.bin
differ
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_current_timezone.json
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_timezone.json
new file mode 100644
index 00000000000..082d7f5bae6
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_timezone.json
@@ -0,0 +1,20 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema":
"struct\u003cd:date,t:timestamp,s:string,x:bigint,wt:struct\u003cstart:timestamp,end:timestamp\u003e\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "current_timezone"
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_current_timezone.proto.bin
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_timezone.proto.bin
new file mode 100644
index 00000000000..a780830516b
Binary files /dev/null and
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_timezone.proto.bin
differ
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_current_user.json
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_user.json
new file mode 100644
index 00000000000..30ddb80f884
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_user.json
@@ -0,0 +1,20 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema":
"struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "current_user"
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_current_user.proto.bin
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_user.proto.bin
new file mode 100644
index 00000000000..0a79078fd70
Binary files /dev/null and
b/connector/connect/common/src/test/resources/query-tests/queries/function_current_user.proto.bin
differ
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_user.json
b/connector/connect/common/src/test/resources/query-tests/queries/function_user.json
new file mode 100644
index 00000000000..aaf3de9ba03
--- /dev/null
+++
b/connector/connect/common/src/test/resources/query-tests/queries/function_user.json
@@ -0,0 +1,20 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema":
"struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "user"
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git
a/connector/connect/common/src/test/resources/query-tests/queries/function_user.proto.bin
b/connector/connect/common/src/test/resources/query-tests/queries/function_user.proto.bin
new file mode 100644
index 00000000000..dbd64cae9f3
Binary files /dev/null and
b/connector/connect/common/src/test/resources/query-tests/queries/function_user.proto.bin
differ
diff --git a/python/docs/source/reference/pyspark.sql/functions.rst
b/python/docs/source/reference/pyspark.sql/functions.rst
index 4b236581f23..8bfad0eb410 100644
--- a/python/docs/source/reference/pyspark.sql/functions.rst
+++ b/python/docs/source/reference/pyspark.sql/functions.rst
@@ -116,8 +116,10 @@ Datetime Functions
:toctree: api/
add_months
+ curdate
current_date
current_timestamp
+ current_timezone
date_add
date_format
date_sub
@@ -389,6 +391,10 @@ Misc Functions
.. autosummary::
:toctree: api/
+ current_catalog
+ current_database
+ current_schema
+ current_user
md5
sha1
sha2
@@ -399,6 +405,7 @@ Misc Functions
raise_error
hll_sketch_estimate
hll_union
+ user
Xml Functions
--------------
diff --git a/python/pyspark/sql/connect/functions.py
b/python/pyspark/sql/connect/functions.py
index abec9c13ae5..04d947c43f2 100644
--- a/python/pyspark/sql/connect/functions.py
+++ b/python/pyspark/sql/connect/functions.py
@@ -2252,6 +2252,13 @@ to_number.__doc__ = pysparkfuncs.to_number.__doc__
# timestamp_seconds, current_timestamp, date_trunc
+def curdate() -> Column:
+ return _invoke_function("curdate")
+
+
+curdate.__doc__ = pysparkfuncs.curdate.__doc__
+
+
def current_date() -> Column:
return _invoke_function("current_date")
@@ -2266,6 +2273,13 @@ def current_timestamp() -> Column:
current_timestamp.__doc__ = pysparkfuncs.current_timestamp.__doc__
+def current_timezone() -> Column:
+ return _invoke_function("current_timezone")
+
+
+current_timezone.__doc__ = pysparkfuncs.current_timezone.__doc__
+
+
def localtimestamp() -> Column:
return _invoke_function("localtimestamp")
@@ -2763,6 +2777,41 @@ hours.__doc__ = pysparkfuncs.hours.__doc__
# Misc Functions
+def current_catalog() -> Column:
+ return _invoke_function("current_catalog")
+
+
+current_catalog.__doc__ = pysparkfuncs.current_catalog.__doc__
+
+
+def current_database() -> Column:
+ return _invoke_function("current_database")
+
+
+current_database.__doc__ = pysparkfuncs.current_database.__doc__
+
+
+def current_schema() -> Column:
+ return _invoke_function("current_schema")
+
+
+current_schema.__doc__ = pysparkfuncs.current_schema.__doc__
+
+
+def current_user() -> Column:
+ return _invoke_function("current_user")
+
+
+current_user.__doc__ = pysparkfuncs.current_user.__doc__
+
+
+def user() -> Column:
+ return _invoke_function("user")
+
+
+user.__doc__ = pysparkfuncs.user.__doc__
+
+
def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] =
None) -> Column:
if errMsg is None:
return _invoke_function_over_columns("assert_true", col)
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index c752c23a0d6..c2e38046038 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -4845,6 +4845,32 @@ def ntile(n: int) -> Column:
# ---------------------- Date/Timestamp functions
------------------------------
+@try_remote_functions
+def curdate() -> Column:
+ """
+ Returns the current date at the start of query evaluation as a
:class:`DateType` column.
+ All calls of current_date within the same query return the same value.
+
+ .. versionadded:: 3.5.0
+
+ Returns
+ -------
+ :class:`~pyspark.sql.Column`
+ current date.
+
+ Examples
+ --------
+ >>> df = spark.range(1)
+ >>> df.select(curdate()).show() # doctest: +SKIP
+ +--------------+
+ |current_date()|
+ +--------------+
+ | 2022-08-26|
+ +--------------+
+ """
+ return _invoke_function("curdate")
+
+
@try_remote_functions
def current_date() -> Column:
"""
@@ -4874,6 +4900,32 @@ def current_date() -> Column:
return _invoke_function("current_date")
+@try_remote_functions
+def current_timezone() -> Column:
+ """
+ Returns the current session local timezone.
+
+ .. versionadded:: 3.5.0
+
+ Returns
+ -------
+ :class:`~pyspark.sql.Column`
+ current session local timezone.
+
+ Examples
+ --------
+ >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
+ >>> spark.range(1).select(current_timezone()).show()
+ +-------------------+
+ | current_timezone()|
+ +-------------------+
+ |America/Los_Angeles|
+ +-------------------+
+ >>> spark.conf.unset("spark.sql.session.timeZone")
+ """
+ return _invoke_function("current_timezone")
+
+
@try_remote_functions
def current_timestamp() -> Column:
"""
@@ -6494,6 +6546,96 @@ def to_timestamp_ntz(
# ---------------------------- misc functions
----------------------------------
+@try_remote_functions
+def current_catalog() -> Column:
+ """Returns the current catalog.
+
+ .. versionadded:: 3.5.0
+
+ Examples
+ --------
+ >>> spark.range(1).select(current_catalog()).show()
+ +-----------------+
+ |current_catalog()|
+ +-----------------+
+ | spark_catalog|
+ +-----------------+
+ """
+ return _invoke_function("current_catalog")
+
+
+@try_remote_functions
+def current_database() -> Column:
+ """Returns the current database.
+
+ .. versionadded:: 3.5.0
+
+ Examples
+ --------
+ >>> spark.range(1).select(current_database()).show()
+ +------------------+
+ |current_database()|
+ +------------------+
+ | default|
+ +------------------+
+ """
+ return _invoke_function("current_database")
+
+
+@try_remote_functions
+def current_schema() -> Column:
+ """Returns the current database.
+
+ .. versionadded:: 3.5.0
+
+ Examples
+ --------
+ >>> spark.range(1).select(current_schema()).show()
+ +------------------+
+ |current_database()|
+ +------------------+
+ | default|
+ +------------------+
+ """
+ return _invoke_function("current_schema")
+
+
+@try_remote_functions
+def current_user() -> Column:
+ """Returns the current database.
+
+ .. versionadded:: 3.5.0
+
+ Examples
+ --------
+ >>> spark.range(1).select(current_user()).show() # doctest: +SKIP
+ +--------------+
+ |current_user()|
+ +--------------+
+ | ruifeng.zheng|
+ +--------------+
+ """
+ return _invoke_function("current_user")
+
+
+@try_remote_functions
+def user() -> Column:
+ """Returns the current database.
+
+ .. versionadded:: 3.5.0
+
+ Examples
+ --------
+ >>> spark.range(1).select(user()).show() # doctest: +SKIP
+ +--------------+
+ |current_user()|
+ +--------------+
+ | ruifeng.zheng|
+ +--------------+
+ """
+ return _invoke_function("user")
+
+
@try_remote_functions
def crc32(col: "ColumnOrName") -> Column:
"""
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 11e2a06bcd3..2e13d844f65 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2851,6 +2851,38 @@ object functions {
// Misc functions
//////////////////////////////////////////////////////////////////////////////////////////////
+ /**
+ * Returns the current catalog.
+ *
+ * @group misc_funcs
+ * @since 3.5.0
+ */
+ def current_catalog(): Column = withExpr { CurrentCatalog() }
+
+ /**
+ * Returns the current database.
+ *
+ * @group misc_funcs
+ * @since 3.5.0
+ */
+ def current_database(): Column = withExpr { CurrentDatabase() }
+
+ /**
+ * Returns the current schema.
+ *
+ * @group misc_funcs
+ * @since 3.5.0
+ */
+ def current_schema(): Column = withExpr { CurrentDatabase() }
+
+ /**
+ * Returns the user name of current execution context.
+ *
+ * @group misc_funcs
+ * @since 3.5.0
+ */
+ def current_user(): Column = withExpr { CurrentUser() }
+
/**
* Calculates the MD5 digest of a binary column and returns the value
* as a 32 character hex string.
@@ -3019,6 +3051,14 @@ object functions {
hll_union(Column(columnName1), Column(columnName2),
allowDifferentLgConfigK)
}
+ /**
+ * Returns the user name of current execution context.
+ *
+ * @group misc_funcs
+ * @since 3.5.0
+ */
+ def user(): Column = withExpr { CurrentUser() }
+
//////////////////////////////////////////////////////////////////////////////////////////////
// String functions
//////////////////////////////////////////////////////////////////////////////////////////////
@@ -3688,6 +3728,15 @@ object functions {
AddMonths(startDate.expr, numMonths.expr)
}
+ /**
+ * Returns the current date at the start of query evaluation as a date
column.
+ * All calls of current_date within the same query return the same value.
+ *
+ * @group datetime_funcs
+ * @since 3.5.0
+ */
+ def curdate(): Column = withExpr { CurrentDate() }
+
/**
* Returns the current date at the start of query evaluation as a date
column.
* All calls of current_date within the same query return the same value.
@@ -3697,6 +3746,14 @@ object functions {
*/
def current_date(): Column = withExpr { CurrentDate() }
+ /**
+ * Returns the current session local timezone.
+ *
+ * @group datetime_funcs
+ * @since 3.5.0
+ */
+ def current_timezone(): Column = withExpr { CurrentTimeZone() }
+
/**
* Returns the current timestamp at the start of query evaluation as a
timestamp column.
* All calls of current_timestamp within the same query return the same
value.
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index 73a0d999c7d..d1946469529 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -5570,6 +5570,21 @@ class DataFrameFunctionsSuite extends QueryTest with
SharedSparkSession {
Seq(Row(Seq("b", "a", "c", null)))
)
}
+
+ test("function current_catalog, current_database, current_schema") {
+ val df = Seq((1, 2), (3, 1)).toDF("a", "b")
+
+ checkAnswer(df.selectExpr("CURRENT_CATALOG()"),
df.select(current_catalog()))
+ checkAnswer(df.selectExpr("CURRENT_DATABASE()"),
df.select(current_database()))
+ checkAnswer(df.selectExpr("CURRENT_SCHEMA()"), df.select(current_schema()))
+ }
+
+ test("function current_user, user") {
+ val df = Seq((1, 2), (3, 1)).toDF("a", "b")
+
+ checkAnswer(df.selectExpr("CURRENT_USER()"), df.select(current_user()))
+ checkAnswer(df.selectExpr("USER()"), df.select(user()))
+ }
}
object DataFrameFunctionsSuite {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index b0462042421..fc3691ca8e6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -65,6 +65,12 @@ class DateFunctionsSuite extends QueryTest with
SharedSparkSession {
)
}
+ test("function current_timezone") {
+ val df = Seq((1, 2), (3, 1)).toDF("a", "b")
+
+ checkAnswer(df.selectExpr("CURRENT_TIMEZONE()"),
df.select(current_timezone()))
+ }
+
test("function current_timestamp and now") {
val df1 = Seq((1, 2), (3, 1)).toDF("a", "b")
checkAnswer(df1.select(count_distinct(current_timestamp())), Row(1))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
