This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new f6df78154bac [SPARK-47035][SS][CONNECT][FOLLOWUP] Additional protobuf change to client side listener f6df78154bac is described below commit f6df78154bac826bd51d2aad185ce02a7efd36b6 Author: Wei Liu <wei....@databricks.com> AuthorDate: Mon Mar 11 09:07:35 2024 +0900 [SPARK-47035][SS][CONNECT][FOLLOWUP] Additional protobuf change to client side listener ### What changes were proposed in this pull request? Followup of previous protocol change https://github.com/apache/spark/pull/45091. Add the request proto `Command` and response proto message to `ExecutePlanResponse` ### Why are the changes needed? Continuation of client side listener for spark connect ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Will be tested in subsequent PR, the proto change itself doesn't do any harm ### Was this patch authored or co-authored using generative AI tooling? No Closes #45444 from WweiL/SPARK-47035-protocol-followup. Authored-by: Wei Liu <wei....@databricks.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../src/main/protobuf/spark/connect/base.proto | 3 + .../src/main/protobuf/spark/connect/commands.proto | 2 + python/pyspark/sql/connect/proto/base_pb2.py | 204 ++++++++++----------- python/pyspark/sql/connect/proto/base_pb2.pyi | 13 ++ python/pyspark/sql/connect/proto/commands_pb2.py | 180 +++++++++--------- python/pyspark/sql/connect/proto/commands_pb2.pyi | 40 +++- 6 files changed, 249 insertions(+), 193 deletions(-) diff --git a/connector/connect/common/src/main/protobuf/spark/connect/base.proto b/connector/connect/common/src/main/protobuf/spark/connect/base.proto index f24ca0a8fc3b..cb9dbe62c193 100644 --- a/connector/connect/common/src/main/protobuf/spark/connect/base.proto +++ b/connector/connect/common/src/main/protobuf/spark/connect/base.proto @@ -357,6 +357,9 @@ message ExecutePlanResponse { // Response for commands on the streaming query manager. StreamingQueryManagerCommandResult streaming_query_manager_command_result = 11; + // Response for commands on the client side streaming query listener. + StreamingQueryListenerEventsResult streaming_query_listener_events_result = 16; + // Response type informing if the stream is complete in reattachable execution. ResultComplete result_complete = 14; diff --git a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto index e845d5f29061..76ac106b1de8 100644 --- a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto +++ b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto @@ -42,6 +42,7 @@ message Command { GetResourcesCommand get_resources_command = 8; StreamingQueryManagerCommand streaming_query_manager_command = 9; CommonInlineUserDefinedTableFunction register_table_function = 10; + StreamingQueryListenerBusCommand streaming_query_listener_bus_command = 11; // This field is used to mark extensions to the protocol. When plugins generate arbitrary // Commands they can add them here. During the planning the correct resolution is done. @@ -456,6 +457,7 @@ message StreamingQueryListenerEvent { message StreamingQueryListenerEventsResult { repeated StreamingQueryListenerEvent events = 1; + optional bool listener_bus_listener_added = 2; } // Command to get the output of 'SparkContext.resources' diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py index 8326ce511d56..1941900ae69d 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.py +++ b/python/pyspark/sql/connect/proto/base_pb2.py @@ -37,7 +37,7 @@ from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17 [...] + b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17 [...] ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -120,105 +120,105 @@ if _descriptor._USE_C_DESCRIPTORS == False: _EXECUTEPLANREQUEST_REQUESTOPTION._serialized_start = 4977 _EXECUTEPLANREQUEST_REQUESTOPTION._serialized_end = 5142 _EXECUTEPLANRESPONSE._serialized_start = 5178 - _EXECUTEPLANRESPONSE._serialized_end = 7253 - _EXECUTEPLANRESPONSE_SQLCOMMANDRESULT._serialized_start = 6389 - _EXECUTEPLANRESPONSE_SQLCOMMANDRESULT._serialized_end = 6460 - _EXECUTEPLANRESPONSE_ARROWBATCH._serialized_start = 6462 - _EXECUTEPLANRESPONSE_ARROWBATCH._serialized_end = 6580 - _EXECUTEPLANRESPONSE_METRICS._serialized_start = 6583 - _EXECUTEPLANRESPONSE_METRICS._serialized_end = 7100 - _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT._serialized_start = 6678 - _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT._serialized_end = 7010 - _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_start = 6887 - _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_end = 7010 - _EXECUTEPLANRESPONSE_METRICS_METRICVALUE._serialized_start = 7012 - _EXECUTEPLANRESPONSE_METRICS_METRICVALUE._serialized_end = 7100 - _EXECUTEPLANRESPONSE_OBSERVEDMETRICS._serialized_start = 7102 - _EXECUTEPLANRESPONSE_OBSERVEDMETRICS._serialized_end = 7218 - _EXECUTEPLANRESPONSE_RESULTCOMPLETE._serialized_start = 7220 - _EXECUTEPLANRESPONSE_RESULTCOMPLETE._serialized_end = 7236 - _KEYVALUE._serialized_start = 7255 - _KEYVALUE._serialized_end = 7320 - _CONFIGREQUEST._serialized_start = 7323 - _CONFIGREQUEST._serialized_end = 8351 - _CONFIGREQUEST_OPERATION._serialized_start = 7543 - _CONFIGREQUEST_OPERATION._serialized_end = 8041 - _CONFIGREQUEST_SET._serialized_start = 8043 - _CONFIGREQUEST_SET._serialized_end = 8095 - _CONFIGREQUEST_GET._serialized_start = 8097 - _CONFIGREQUEST_GET._serialized_end = 8122 - _CONFIGREQUEST_GETWITHDEFAULT._serialized_start = 8124 - _CONFIGREQUEST_GETWITHDEFAULT._serialized_end = 8187 - _CONFIGREQUEST_GETOPTION._serialized_start = 8189 - _CONFIGREQUEST_GETOPTION._serialized_end = 8220 - _CONFIGREQUEST_GETALL._serialized_start = 8222 - _CONFIGREQUEST_GETALL._serialized_end = 8270 - _CONFIGREQUEST_UNSET._serialized_start = 8272 - _CONFIGREQUEST_UNSET._serialized_end = 8299 - _CONFIGREQUEST_ISMODIFIABLE._serialized_start = 8301 - _CONFIGREQUEST_ISMODIFIABLE._serialized_end = 8335 - _CONFIGRESPONSE._serialized_start = 8354 - _CONFIGRESPONSE._serialized_end = 8529 - _ADDARTIFACTSREQUEST._serialized_start = 8532 - _ADDARTIFACTSREQUEST._serialized_end = 9403 - _ADDARTIFACTSREQUEST_ARTIFACTCHUNK._serialized_start = 8919 - _ADDARTIFACTSREQUEST_ARTIFACTCHUNK._serialized_end = 8972 - _ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT._serialized_start = 8974 - _ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT._serialized_end = 9085 - _ADDARTIFACTSREQUEST_BATCH._serialized_start = 9087 - _ADDARTIFACTSREQUEST_BATCH._serialized_end = 9180 - _ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT._serialized_start = 9183 - _ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT._serialized_end = 9376 - _ADDARTIFACTSRESPONSE._serialized_start = 9406 - _ADDARTIFACTSRESPONSE._serialized_end = 9678 - _ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY._serialized_start = 9597 - _ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY._serialized_end = 9678 - _ARTIFACTSTATUSESREQUEST._serialized_start = 9681 - _ARTIFACTSTATUSESREQUEST._serialized_end = 9876 - _ARTIFACTSTATUSESRESPONSE._serialized_start = 9879 - _ARTIFACTSTATUSESRESPONSE._serialized_end = 10231 - _ARTIFACTSTATUSESRESPONSE_STATUSESENTRY._serialized_start = 10074 - _ARTIFACTSTATUSESRESPONSE_STATUSESENTRY._serialized_end = 10189 - _ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS._serialized_start = 10191 - _ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS._serialized_end = 10231 - _INTERRUPTREQUEST._serialized_start = 10234 - _INTERRUPTREQUEST._serialized_end = 10706 - _INTERRUPTREQUEST_INTERRUPTTYPE._serialized_start = 10549 - _INTERRUPTREQUEST_INTERRUPTTYPE._serialized_end = 10677 - _INTERRUPTRESPONSE._serialized_start = 10709 - _INTERRUPTRESPONSE._serialized_end = 10853 - _REATTACHOPTIONS._serialized_start = 10855 - _REATTACHOPTIONS._serialized_end = 10908 - _REATTACHEXECUTEREQUEST._serialized_start = 10911 - _REATTACHEXECUTEREQUEST._serialized_end = 11186 - _RELEASEEXECUTEREQUEST._serialized_start = 11189 - _RELEASEEXECUTEREQUEST._serialized_end = 11643 - _RELEASEEXECUTEREQUEST_RELEASEALL._serialized_start = 11555 - _RELEASEEXECUTEREQUEST_RELEASEALL._serialized_end = 11567 - _RELEASEEXECUTEREQUEST_RELEASEUNTIL._serialized_start = 11569 - _RELEASEEXECUTEREQUEST_RELEASEUNTIL._serialized_end = 11616 - _RELEASEEXECUTERESPONSE._serialized_start = 11646 - _RELEASEEXECUTERESPONSE._serialized_end = 11811 - _RELEASESESSIONREQUEST._serialized_start = 11814 - _RELEASESESSIONREQUEST._serialized_end = 11985 - _RELEASESESSIONRESPONSE._serialized_start = 11987 - _RELEASESESSIONRESPONSE._serialized_end = 12095 - _FETCHERRORDETAILSREQUEST._serialized_start = 12098 - _FETCHERRORDETAILSREQUEST._serialized_end = 12299 - _FETCHERRORDETAILSRESPONSE._serialized_start = 12302 - _FETCHERRORDETAILSRESPONSE._serialized_end = 13857 - _FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT._serialized_start = 12531 - _FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT._serialized_end = 12705 - _FETCHERRORDETAILSRESPONSE_QUERYCONTEXT._serialized_start = 12708 - _FETCHERRORDETAILSRESPONSE_QUERYCONTEXT._serialized_end = 13076 - _FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE._serialized_start = 13039 - _FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE._serialized_end = 13076 - _FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE._serialized_start = 13079 - _FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE._serialized_end = 13488 - _FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY._serialized_start = 13390 - _FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY._serialized_end = 13458 - _FETCHERRORDETAILSRESPONSE_ERROR._serialized_start = 13491 - _FETCHERRORDETAILSRESPONSE_ERROR._serialized_end = 13838 - _SPARKCONNECTSERVICE._serialized_start = 13860 - _SPARKCONNECTSERVICE._serialized_end = 14806 + _EXECUTEPLANRESPONSE._serialized_end = 7391 + _EXECUTEPLANRESPONSE_SQLCOMMANDRESULT._serialized_start = 6527 + _EXECUTEPLANRESPONSE_SQLCOMMANDRESULT._serialized_end = 6598 + _EXECUTEPLANRESPONSE_ARROWBATCH._serialized_start = 6600 + _EXECUTEPLANRESPONSE_ARROWBATCH._serialized_end = 6718 + _EXECUTEPLANRESPONSE_METRICS._serialized_start = 6721 + _EXECUTEPLANRESPONSE_METRICS._serialized_end = 7238 + _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT._serialized_start = 6816 + _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT._serialized_end = 7148 + _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_start = 7025 + _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_end = 7148 + _EXECUTEPLANRESPONSE_METRICS_METRICVALUE._serialized_start = 7150 + _EXECUTEPLANRESPONSE_METRICS_METRICVALUE._serialized_end = 7238 + _EXECUTEPLANRESPONSE_OBSERVEDMETRICS._serialized_start = 7240 + _EXECUTEPLANRESPONSE_OBSERVEDMETRICS._serialized_end = 7356 + _EXECUTEPLANRESPONSE_RESULTCOMPLETE._serialized_start = 7358 + _EXECUTEPLANRESPONSE_RESULTCOMPLETE._serialized_end = 7374 + _KEYVALUE._serialized_start = 7393 + _KEYVALUE._serialized_end = 7458 + _CONFIGREQUEST._serialized_start = 7461 + _CONFIGREQUEST._serialized_end = 8489 + _CONFIGREQUEST_OPERATION._serialized_start = 7681 + _CONFIGREQUEST_OPERATION._serialized_end = 8179 + _CONFIGREQUEST_SET._serialized_start = 8181 + _CONFIGREQUEST_SET._serialized_end = 8233 + _CONFIGREQUEST_GET._serialized_start = 8235 + _CONFIGREQUEST_GET._serialized_end = 8260 + _CONFIGREQUEST_GETWITHDEFAULT._serialized_start = 8262 + _CONFIGREQUEST_GETWITHDEFAULT._serialized_end = 8325 + _CONFIGREQUEST_GETOPTION._serialized_start = 8327 + _CONFIGREQUEST_GETOPTION._serialized_end = 8358 + _CONFIGREQUEST_GETALL._serialized_start = 8360 + _CONFIGREQUEST_GETALL._serialized_end = 8408 + _CONFIGREQUEST_UNSET._serialized_start = 8410 + _CONFIGREQUEST_UNSET._serialized_end = 8437 + _CONFIGREQUEST_ISMODIFIABLE._serialized_start = 8439 + _CONFIGREQUEST_ISMODIFIABLE._serialized_end = 8473 + _CONFIGRESPONSE._serialized_start = 8492 + _CONFIGRESPONSE._serialized_end = 8667 + _ADDARTIFACTSREQUEST._serialized_start = 8670 + _ADDARTIFACTSREQUEST._serialized_end = 9541 + _ADDARTIFACTSREQUEST_ARTIFACTCHUNK._serialized_start = 9057 + _ADDARTIFACTSREQUEST_ARTIFACTCHUNK._serialized_end = 9110 + _ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT._serialized_start = 9112 + _ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT._serialized_end = 9223 + _ADDARTIFACTSREQUEST_BATCH._serialized_start = 9225 + _ADDARTIFACTSREQUEST_BATCH._serialized_end = 9318 + _ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT._serialized_start = 9321 + _ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT._serialized_end = 9514 + _ADDARTIFACTSRESPONSE._serialized_start = 9544 + _ADDARTIFACTSRESPONSE._serialized_end = 9816 + _ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY._serialized_start = 9735 + _ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY._serialized_end = 9816 + _ARTIFACTSTATUSESREQUEST._serialized_start = 9819 + _ARTIFACTSTATUSESREQUEST._serialized_end = 10014 + _ARTIFACTSTATUSESRESPONSE._serialized_start = 10017 + _ARTIFACTSTATUSESRESPONSE._serialized_end = 10369 + _ARTIFACTSTATUSESRESPONSE_STATUSESENTRY._serialized_start = 10212 + _ARTIFACTSTATUSESRESPONSE_STATUSESENTRY._serialized_end = 10327 + _ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS._serialized_start = 10329 + _ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS._serialized_end = 10369 + _INTERRUPTREQUEST._serialized_start = 10372 + _INTERRUPTREQUEST._serialized_end = 10844 + _INTERRUPTREQUEST_INTERRUPTTYPE._serialized_start = 10687 + _INTERRUPTREQUEST_INTERRUPTTYPE._serialized_end = 10815 + _INTERRUPTRESPONSE._serialized_start = 10847 + _INTERRUPTRESPONSE._serialized_end = 10991 + _REATTACHOPTIONS._serialized_start = 10993 + _REATTACHOPTIONS._serialized_end = 11046 + _REATTACHEXECUTEREQUEST._serialized_start = 11049 + _REATTACHEXECUTEREQUEST._serialized_end = 11324 + _RELEASEEXECUTEREQUEST._serialized_start = 11327 + _RELEASEEXECUTEREQUEST._serialized_end = 11781 + _RELEASEEXECUTEREQUEST_RELEASEALL._serialized_start = 11693 + _RELEASEEXECUTEREQUEST_RELEASEALL._serialized_end = 11705 + _RELEASEEXECUTEREQUEST_RELEASEUNTIL._serialized_start = 11707 + _RELEASEEXECUTEREQUEST_RELEASEUNTIL._serialized_end = 11754 + _RELEASEEXECUTERESPONSE._serialized_start = 11784 + _RELEASEEXECUTERESPONSE._serialized_end = 11949 + _RELEASESESSIONREQUEST._serialized_start = 11952 + _RELEASESESSIONREQUEST._serialized_end = 12123 + _RELEASESESSIONRESPONSE._serialized_start = 12125 + _RELEASESESSIONRESPONSE._serialized_end = 12233 + _FETCHERRORDETAILSREQUEST._serialized_start = 12236 + _FETCHERRORDETAILSREQUEST._serialized_end = 12437 + _FETCHERRORDETAILSRESPONSE._serialized_start = 12440 + _FETCHERRORDETAILSRESPONSE._serialized_end = 13995 + _FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT._serialized_start = 12669 + _FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT._serialized_end = 12843 + _FETCHERRORDETAILSRESPONSE_QUERYCONTEXT._serialized_start = 12846 + _FETCHERRORDETAILSRESPONSE_QUERYCONTEXT._serialized_end = 13214 + _FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE._serialized_start = 13177 + _FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE._serialized_end = 13214 + _FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE._serialized_start = 13217 + _FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE._serialized_end = 13626 + _FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY._serialized_start = 13528 + _FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY._serialized_end = 13596 + _FETCHERRORDETAILSRESPONSE_ERROR._serialized_start = 13629 + _FETCHERRORDETAILSRESPONSE_ERROR._serialized_end = 13976 + _SPARKCONNECTSERVICE._serialized_start = 13998 + _SPARKCONNECTSERVICE._serialized_end = 14944 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi index e4ed03dc6945..5ed2d207aca5 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.pyi +++ b/python/pyspark/sql/connect/proto/base_pb2.pyi @@ -1410,6 +1410,7 @@ class ExecutePlanResponse(google.protobuf.message.Message): STREAMING_QUERY_COMMAND_RESULT_FIELD_NUMBER: builtins.int GET_RESOURCES_COMMAND_RESULT_FIELD_NUMBER: builtins.int STREAMING_QUERY_MANAGER_COMMAND_RESULT_FIELD_NUMBER: builtins.int + STREAMING_QUERY_LISTENER_EVENTS_RESULT_FIELD_NUMBER: builtins.int RESULT_COMPLETE_FIELD_NUMBER: builtins.int EXTENSION_FIELD_NUMBER: builtins.int METRICS_FIELD_NUMBER: builtins.int @@ -1456,6 +1457,11 @@ class ExecutePlanResponse(google.protobuf.message.Message): ) -> pyspark.sql.connect.proto.commands_pb2.StreamingQueryManagerCommandResult: """Response for commands on the streaming query manager.""" @property + def streaming_query_listener_events_result( + self, + ) -> pyspark.sql.connect.proto.commands_pb2.StreamingQueryListenerEventsResult: + """Response for commands on the client side streaming query listener.""" + @property def result_complete(self) -> global___ExecutePlanResponse.ResultComplete: """Response type informing if the stream is complete in reattachable execution.""" @property @@ -1493,6 +1499,8 @@ class ExecutePlanResponse(google.protobuf.message.Message): | None = ..., streaming_query_manager_command_result: pyspark.sql.connect.proto.commands_pb2.StreamingQueryManagerCommandResult | None = ..., + streaming_query_listener_events_result: pyspark.sql.connect.proto.commands_pb2.StreamingQueryListenerEventsResult + | None = ..., result_complete: global___ExecutePlanResponse.ResultComplete | None = ..., extension: google.protobuf.any_pb2.Any | None = ..., metrics: global___ExecutePlanResponse.Metrics | None = ..., @@ -1521,6 +1529,8 @@ class ExecutePlanResponse(google.protobuf.message.Message): b"sql_command_result", "streaming_query_command_result", b"streaming_query_command_result", + "streaming_query_listener_events_result", + b"streaming_query_listener_events_result", "streaming_query_manager_command_result", b"streaming_query_manager_command_result", "write_stream_operation_start_result", @@ -1558,6 +1568,8 @@ class ExecutePlanResponse(google.protobuf.message.Message): b"sql_command_result", "streaming_query_command_result", b"streaming_query_command_result", + "streaming_query_listener_events_result", + b"streaming_query_listener_events_result", "streaming_query_manager_command_result", b"streaming_query_manager_command_result", "write_stream_operation_start_result", @@ -1574,6 +1586,7 @@ class ExecutePlanResponse(google.protobuf.message.Message): "streaming_query_command_result", "get_resources_command_result", "streaming_query_manager_command_result", + "streaming_query_listener_events_result", "result_complete", "extension", ] diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py index cfeb57a0c56a..45e276bf96e7 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.py +++ b/python/pyspark/sql/connect/proto/commands_pb2.py @@ -35,7 +35,7 @@ from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_rel DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto"\xf5\x07\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x [...] + b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto"\xf9\x08\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x [...] ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -63,94 +63,94 @@ if _descriptor._USE_C_DESCRIPTORS == False: _WRITESTREAMOPERATIONSTART_OPTIONSENTRY._serialized_options = b"8\001" _GETRESOURCESCOMMANDRESULT_RESOURCESENTRY._options = None _GETRESOURCESCOMMANDRESULT_RESOURCESENTRY._serialized_options = b"8\001" - _STREAMINGQUERYEVENTTYPE._serialized_start = 9409 - _STREAMINGQUERYEVENTTYPE._serialized_end = 9542 + _STREAMINGQUERYEVENTTYPE._serialized_start = 9642 + _STREAMINGQUERYEVENTTYPE._serialized_end = 9775 _COMMAND._serialized_start = 167 - _COMMAND._serialized_end = 1180 - _SQLCOMMAND._serialized_start = 1183 - _SQLCOMMAND._serialized_end = 1678 - _SQLCOMMAND_ARGSENTRY._serialized_start = 1494 - _SQLCOMMAND_ARGSENTRY._serialized_end = 1584 - _SQLCOMMAND_NAMEDARGUMENTSENTRY._serialized_start = 1586 - _SQLCOMMAND_NAMEDARGUMENTSENTRY._serialized_end = 1678 - _CREATEDATAFRAMEVIEWCOMMAND._serialized_start = 1681 - _CREATEDATAFRAMEVIEWCOMMAND._serialized_end = 1831 - _WRITEOPERATION._serialized_start = 1834 - _WRITEOPERATION._serialized_end = 2932 - _WRITEOPERATION_OPTIONSENTRY._serialized_start = 2356 - _WRITEOPERATION_OPTIONSENTRY._serialized_end = 2414 - _WRITEOPERATION_SAVETABLE._serialized_start = 2417 - _WRITEOPERATION_SAVETABLE._serialized_end = 2675 - _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_start = 2551 - _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_end = 2675 - _WRITEOPERATION_BUCKETBY._serialized_start = 2677 - _WRITEOPERATION_BUCKETBY._serialized_end = 2768 - _WRITEOPERATION_SAVEMODE._serialized_start = 2771 - _WRITEOPERATION_SAVEMODE._serialized_end = 2908 - _WRITEOPERATIONV2._serialized_start = 2935 - _WRITEOPERATIONV2._serialized_end = 3795 - _WRITEOPERATIONV2_OPTIONSENTRY._serialized_start = 2356 - _WRITEOPERATIONV2_OPTIONSENTRY._serialized_end = 2414 - _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_start = 3554 - _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_end = 3620 - _WRITEOPERATIONV2_MODE._serialized_start = 3623 - _WRITEOPERATIONV2_MODE._serialized_end = 3782 - _WRITESTREAMOPERATIONSTART._serialized_start = 3798 - _WRITESTREAMOPERATIONSTART._serialized_end = 4598 - _WRITESTREAMOPERATIONSTART_OPTIONSENTRY._serialized_start = 2356 - _WRITESTREAMOPERATIONSTART_OPTIONSENTRY._serialized_end = 2414 - _STREAMINGFOREACHFUNCTION._serialized_start = 4601 - _STREAMINGFOREACHFUNCTION._serialized_end = 4780 - _WRITESTREAMOPERATIONSTARTRESULT._serialized_start = 4783 - _WRITESTREAMOPERATIONSTARTRESULT._serialized_end = 4995 - _STREAMINGQUERYINSTANCEID._serialized_start = 4997 - _STREAMINGQUERYINSTANCEID._serialized_end = 5062 - _STREAMINGQUERYCOMMAND._serialized_start = 5065 - _STREAMINGQUERYCOMMAND._serialized_end = 5697 - _STREAMINGQUERYCOMMAND_EXPLAINCOMMAND._serialized_start = 5564 - _STREAMINGQUERYCOMMAND_EXPLAINCOMMAND._serialized_end = 5608 - _STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND._serialized_start = 5610 - _STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND._serialized_end = 5686 - _STREAMINGQUERYCOMMANDRESULT._serialized_start = 5700 - _STREAMINGQUERYCOMMANDRESULT._serialized_end = 6841 - _STREAMINGQUERYCOMMANDRESULT_STATUSRESULT._serialized_start = 6283 - _STREAMINGQUERYCOMMANDRESULT_STATUSRESULT._serialized_end = 6453 - _STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT._serialized_start = 6455 - _STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT._serialized_end = 6527 - _STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT._serialized_start = 6529 - _STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT._serialized_end = 6568 - _STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT._serialized_start = 6571 - _STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT._serialized_end = 6768 - _STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT._serialized_start = 6770 - _STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT._serialized_end = 6826 - _STREAMINGQUERYMANAGERCOMMAND._serialized_start = 6844 - _STREAMINGQUERYMANAGERCOMMAND._serialized_end = 7673 - _STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND._serialized_start = 7375 - _STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND._serialized_end = 7454 - _STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND._serialized_start = 7457 - _STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND._serialized_end = 7662 - _STREAMINGQUERYMANAGERCOMMANDRESULT._serialized_start = 7676 - _STREAMINGQUERYMANAGERCOMMANDRESULT._serialized_end = 8752 - _STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT._serialized_start = 8284 - _STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT._serialized_end = 8411 - _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE._serialized_start = 8413 - _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE._serialized_end = 8528 - _STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT._serialized_start = 8530 - _STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT._serialized_end = 8589 - _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE._serialized_start = 8591 - _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE._serialized_end = 8666 - _STREAMINGQUERYMANAGERCOMMANDRESULT_LISTSTREAMINGQUERYLISTENERRESULT._serialized_start = 8668 - _STREAMINGQUERYMANAGERCOMMANDRESULT_LISTSTREAMINGQUERYLISTENERRESULT._serialized_end = 8737 - _STREAMINGQUERYLISTENERBUSCOMMAND._serialized_start = 8755 - _STREAMINGQUERYLISTENERBUSCOMMAND._serialized_end = 8928 - _STREAMINGQUERYLISTENEREVENT._serialized_start = 8931 - _STREAMINGQUERYLISTENEREVENT._serialized_end = 9062 - _STREAMINGQUERYLISTENEREVENTSRESULT._serialized_start = 9064 - _STREAMINGQUERYLISTENEREVENTSRESULT._serialized_end = 9168 - _GETRESOURCESCOMMAND._serialized_start = 9170 - _GETRESOURCESCOMMAND._serialized_end = 9191 - _GETRESOURCESCOMMANDRESULT._serialized_start = 9194 - _GETRESOURCESCOMMANDRESULT._serialized_end = 9406 - _GETRESOURCESCOMMANDRESULT_RESOURCESENTRY._serialized_start = 9310 - _GETRESOURCESCOMMANDRESULT_RESOURCESENTRY._serialized_end = 9406 + _COMMAND._serialized_end = 1312 + _SQLCOMMAND._serialized_start = 1315 + _SQLCOMMAND._serialized_end = 1810 + _SQLCOMMAND_ARGSENTRY._serialized_start = 1626 + _SQLCOMMAND_ARGSENTRY._serialized_end = 1716 + _SQLCOMMAND_NAMEDARGUMENTSENTRY._serialized_start = 1718 + _SQLCOMMAND_NAMEDARGUMENTSENTRY._serialized_end = 1810 + _CREATEDATAFRAMEVIEWCOMMAND._serialized_start = 1813 + _CREATEDATAFRAMEVIEWCOMMAND._serialized_end = 1963 + _WRITEOPERATION._serialized_start = 1966 + _WRITEOPERATION._serialized_end = 3064 + _WRITEOPERATION_OPTIONSENTRY._serialized_start = 2488 + _WRITEOPERATION_OPTIONSENTRY._serialized_end = 2546 + _WRITEOPERATION_SAVETABLE._serialized_start = 2549 + _WRITEOPERATION_SAVETABLE._serialized_end = 2807 + _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_start = 2683 + _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_end = 2807 + _WRITEOPERATION_BUCKETBY._serialized_start = 2809 + _WRITEOPERATION_BUCKETBY._serialized_end = 2900 + _WRITEOPERATION_SAVEMODE._serialized_start = 2903 + _WRITEOPERATION_SAVEMODE._serialized_end = 3040 + _WRITEOPERATIONV2._serialized_start = 3067 + _WRITEOPERATIONV2._serialized_end = 3927 + _WRITEOPERATIONV2_OPTIONSENTRY._serialized_start = 2488 + _WRITEOPERATIONV2_OPTIONSENTRY._serialized_end = 2546 + _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_start = 3686 + _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_end = 3752 + _WRITEOPERATIONV2_MODE._serialized_start = 3755 + _WRITEOPERATIONV2_MODE._serialized_end = 3914 + _WRITESTREAMOPERATIONSTART._serialized_start = 3930 + _WRITESTREAMOPERATIONSTART._serialized_end = 4730 + _WRITESTREAMOPERATIONSTART_OPTIONSENTRY._serialized_start = 2488 + _WRITESTREAMOPERATIONSTART_OPTIONSENTRY._serialized_end = 2546 + _STREAMINGFOREACHFUNCTION._serialized_start = 4733 + _STREAMINGFOREACHFUNCTION._serialized_end = 4912 + _WRITESTREAMOPERATIONSTARTRESULT._serialized_start = 4915 + _WRITESTREAMOPERATIONSTARTRESULT._serialized_end = 5127 + _STREAMINGQUERYINSTANCEID._serialized_start = 5129 + _STREAMINGQUERYINSTANCEID._serialized_end = 5194 + _STREAMINGQUERYCOMMAND._serialized_start = 5197 + _STREAMINGQUERYCOMMAND._serialized_end = 5829 + _STREAMINGQUERYCOMMAND_EXPLAINCOMMAND._serialized_start = 5696 + _STREAMINGQUERYCOMMAND_EXPLAINCOMMAND._serialized_end = 5740 + _STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND._serialized_start = 5742 + _STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND._serialized_end = 5818 + _STREAMINGQUERYCOMMANDRESULT._serialized_start = 5832 + _STREAMINGQUERYCOMMANDRESULT._serialized_end = 6973 + _STREAMINGQUERYCOMMANDRESULT_STATUSRESULT._serialized_start = 6415 + _STREAMINGQUERYCOMMANDRESULT_STATUSRESULT._serialized_end = 6585 + _STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT._serialized_start = 6587 + _STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT._serialized_end = 6659 + _STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT._serialized_start = 6661 + _STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT._serialized_end = 6700 + _STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT._serialized_start = 6703 + _STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT._serialized_end = 6900 + _STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT._serialized_start = 6902 + _STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT._serialized_end = 6958 + _STREAMINGQUERYMANAGERCOMMAND._serialized_start = 6976 + _STREAMINGQUERYMANAGERCOMMAND._serialized_end = 7805 + _STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND._serialized_start = 7507 + _STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND._serialized_end = 7586 + _STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND._serialized_start = 7589 + _STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND._serialized_end = 7794 + _STREAMINGQUERYMANAGERCOMMANDRESULT._serialized_start = 7808 + _STREAMINGQUERYMANAGERCOMMANDRESULT._serialized_end = 8884 + _STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT._serialized_start = 8416 + _STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT._serialized_end = 8543 + _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE._serialized_start = 8545 + _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE._serialized_end = 8660 + _STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT._serialized_start = 8662 + _STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT._serialized_end = 8721 + _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE._serialized_start = 8723 + _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE._serialized_end = 8798 + _STREAMINGQUERYMANAGERCOMMANDRESULT_LISTSTREAMINGQUERYLISTENERRESULT._serialized_start = 8800 + _STREAMINGQUERYMANAGERCOMMANDRESULT_LISTSTREAMINGQUERYLISTENERRESULT._serialized_end = 8869 + _STREAMINGQUERYLISTENERBUSCOMMAND._serialized_start = 8887 + _STREAMINGQUERYLISTENERBUSCOMMAND._serialized_end = 9060 + _STREAMINGQUERYLISTENEREVENT._serialized_start = 9063 + _STREAMINGQUERYLISTENEREVENT._serialized_end = 9194 + _STREAMINGQUERYLISTENEREVENTSRESULT._serialized_start = 9197 + _STREAMINGQUERYLISTENEREVENTSRESULT._serialized_end = 9401 + _GETRESOURCESCOMMAND._serialized_start = 9403 + _GETRESOURCESCOMMAND._serialized_end = 9424 + _GETRESOURCESCOMMANDRESULT._serialized_start = 9427 + _GETRESOURCESCOMMANDRESULT._serialized_end = 9639 + _GETRESOURCESCOMMANDRESULT_RESOURCESENTRY._serialized_start = 9543 + _GETRESOURCESCOMMANDRESULT_RESOURCESENTRY._serialized_end = 9639 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi index e5b09bf0b286..fbbec4f81d2d 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.pyi +++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi @@ -98,6 +98,7 @@ class Command(google.protobuf.message.Message): GET_RESOURCES_COMMAND_FIELD_NUMBER: builtins.int STREAMING_QUERY_MANAGER_COMMAND_FIELD_NUMBER: builtins.int REGISTER_TABLE_FUNCTION_FIELD_NUMBER: builtins.int + STREAMING_QUERY_LISTENER_BUS_COMMAND_FIELD_NUMBER: builtins.int EXTENSION_FIELD_NUMBER: builtins.int @property def register_function( @@ -124,6 +125,8 @@ class Command(google.protobuf.message.Message): self, ) -> pyspark.sql.connect.proto.relations_pb2.CommonInlineUserDefinedTableFunction: ... @property + def streaming_query_listener_bus_command(self) -> global___StreamingQueryListenerBusCommand: ... + @property def extension(self) -> google.protobuf.any_pb2.Any: """This field is used to mark extensions to the protocol. When plugins generate arbitrary Commands they can add them here. During the planning the correct resolution is done. @@ -143,6 +146,8 @@ class Command(google.protobuf.message.Message): streaming_query_manager_command: global___StreamingQueryManagerCommand | None = ..., register_table_function: pyspark.sql.connect.proto.relations_pb2.CommonInlineUserDefinedTableFunction | None = ..., + streaming_query_listener_bus_command: global___StreamingQueryListenerBusCommand + | None = ..., extension: google.protobuf.any_pb2.Any | None = ..., ) -> None: ... def HasField( @@ -164,6 +169,8 @@ class Command(google.protobuf.message.Message): b"sql_command", "streaming_query_command", b"streaming_query_command", + "streaming_query_listener_bus_command", + b"streaming_query_listener_bus_command", "streaming_query_manager_command", b"streaming_query_manager_command", "write_operation", @@ -193,6 +200,8 @@ class Command(google.protobuf.message.Message): b"sql_command", "streaming_query_command", b"streaming_query_command", + "streaming_query_listener_bus_command", + b"streaming_query_listener_bus_command", "streaming_query_manager_command", b"streaming_query_manager_command", "write_operation", @@ -217,6 +226,7 @@ class Command(google.protobuf.message.Message): "get_resources_command", "streaming_query_manager_command", "register_table_function", + "streaming_query_listener_bus_command", "extension", ] | None @@ -1938,18 +1948,46 @@ class StreamingQueryListenerEventsResult(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor EVENTS_FIELD_NUMBER: builtins.int + LISTENER_BUS_LISTENER_ADDED_FIELD_NUMBER: builtins.int @property def events( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ global___StreamingQueryListenerEvent ]: ... + listener_bus_listener_added: builtins.bool def __init__( self, *, events: collections.abc.Iterable[global___StreamingQueryListenerEvent] | None = ..., + listener_bus_listener_added: builtins.bool | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_listener_bus_listener_added", + b"_listener_bus_listener_added", + "listener_bus_listener_added", + b"listener_bus_listener_added", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_listener_bus_listener_added", + b"_listener_bus_listener_added", + "events", + b"events", + "listener_bus_listener_added", + b"listener_bus_listener_added", + ], ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["events", b"events"]) -> None: ... + def WhichOneof( + self, + oneof_group: typing_extensions.Literal[ + "_listener_bus_listener_added", b"_listener_bus_listener_added" + ], + ) -> typing_extensions.Literal["listener_bus_listener_added"] | None: ... global___StreamingQueryListenerEventsResult = StreamingQueryListenerEventsResult --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org