This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c7878850f55 [SPARK-41306][CONNECT] Improve Connect Expression proto
documentation
c7878850f55 is described below
commit c7878850f55643fa061e25f9ffba60edf4f37275
Author: Rui Wang <rui.w...@databricks.com>
AuthorDate: Tue Nov 29 09:02:03 2022 +0800
[SPARK-41306][CONNECT] Improve Connect Expression proto documentation
### What changes were proposed in this pull request?
Improve Connect Expression proto documentation by following
https://github.com/apache/spark/blob/master/connector/connect/docs/adding-proto-messages.md.
### Why are the changes needed?
Proto Documentation.
### Does this PR introduce _any_ user-facing change?
NO
### How was this patch tested?
N/A
Closes #38825 from amaliujia/improve_proto_expr_documentation.
Authored-by: Rui Wang <rui.w...@databricks.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../main/protobuf/spark/connect/expressions.proto | 14 +++++++++++++-
.../pyspark/sql/connect/proto/expressions_pb2.pyi | 21 ++++++++++++++++-----
2 files changed, 29 insertions(+), 6 deletions(-)
diff --git
a/connector/connect/src/main/protobuf/spark/connect/expressions.proto
b/connector/connect/src/main/protobuf/spark/connect/expressions.proto
index 85049f1d14b..595758f0443 100644
--- a/connector/connect/src/main/protobuf/spark/connect/expressions.proto
+++ b/connector/connect/src/main/protobuf/spark/connect/expressions.proto
@@ -142,18 +142,24 @@ message Expression {
// An unresolved attribute that is not explicitly bound to a specific
column, but the column
// is resolved during analysis by name.
message UnresolvedAttribute {
+ // (Required) An identifier that will be parsed by Catalyst parser. This
should follow the
+ // Spark SQL identifier syntax.
string unparsed_identifier = 1;
}
// An unresolved function is not explicitly bound to one explicit function,
but the function
// is resolved during analysis following Sparks name resolution rules.
message UnresolvedFunction {
+ // (Required) Names parts for the unresolved function.
repeated string parts = 1;
+
+ // (Optional) Function arguments. Empty arguments are allowed.
repeated Expression arguments = 2;
}
// Expression as string.
message ExpressionString {
+ // (Required) A SQL expression that will be parsed by Catalyst parser.
string expression = 1;
}
@@ -162,9 +168,15 @@ message Expression {
}
message Alias {
+ // (Required) The expression that alias will be added on.
Expression expr = 1;
+
+ // (Required) a list of name parts for the alias.
+ //
+ // Scalar columns only has one name that presents.
repeated string name = 2;
- // Alias metadata expressed as a JSON map.
+
+ // (Optional) Alias metadata expressed as a JSON map.
optional string metadata = 3;
}
}
diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.pyi
b/python/pyspark/sql/connect/proto/expressions_pb2.pyi
index bf2dc06b685..23ef99c6c48 100644
--- a/python/pyspark/sql/connect/proto/expressions_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/expressions_pb2.pyi
@@ -538,6 +538,9 @@ class Expression(google.protobuf.message.Message):
UNPARSED_IDENTIFIER_FIELD_NUMBER: builtins.int
unparsed_identifier: builtins.str
+ """(Required) An identifier that will be parsed by Catalyst parser.
This should follow the
+ Spark SQL identifier syntax.
+ """
def __init__(
self,
*,
@@ -560,13 +563,15 @@ class Expression(google.protobuf.message.Message):
@property
def parts(
self,
- ) ->
google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
...
+ ) ->
google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
+ """(Required) Names parts for the unresolved function."""
@property
def arguments(
self,
) ->
google.protobuf.internal.containers.RepeatedCompositeFieldContainer[
global___Expression
- ]: ...
+ ]:
+ """(Optional) Function arguments. Empty arguments are allowed."""
def __init__(
self,
*,
@@ -585,6 +590,7 @@ class Expression(google.protobuf.message.Message):
EXPRESSION_FIELD_NUMBER: builtins.int
expression: builtins.str
+ """(Required) A SQL expression that will be parsed by Catalyst
parser."""
def __init__(
self,
*,
@@ -610,13 +616,18 @@ class Expression(google.protobuf.message.Message):
NAME_FIELD_NUMBER: builtins.int
METADATA_FIELD_NUMBER: builtins.int
@property
- def expr(self) -> global___Expression: ...
+ def expr(self) -> global___Expression:
+ """(Required) The expression that alias will be added on."""
@property
def name(
self,
- ) ->
google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
...
+ ) ->
google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
+ """(Required) a list of name parts for the alias.
+
+ Scalar columns only has one name that presents.
+ """
metadata: builtins.str
- """Alias metadata expressed as a JSON map."""
+ """(Optional) Alias metadata expressed as a JSON map."""
def __init__(
self,
*,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
