olayiwola-gbenga commented on code in PR #74664:
URL: https://github.com/apache/airflow/pull/74664#discussion_r4527151458
##########
providers/google/src/airflow/providers/google/firebase/hooks/firestore.py:
##########
@@ -84,9 +91,188 @@ def get_conn(self):
self._conn = build_from_document(non_authorized_conn._rootDesc,
http=http_authorized)
return self._conn
+ def get_operation(self, operation_name: str) -> dict[str, Any]:
+ """
+ Retrieve the current state of a long-running Firestore operation.
+
+ :param operation_name: The resource name of the operation.
+ """
+ service = self.get_conn()
+ return (
+ service.projects()
+ .databases()
+ .operations()
+ .get(name=operation_name)
+ .execute(num_retries=self.num_retries)
+ )
+
+ @GoogleBaseHook.fallback_to_default_project_id
+ def list_operations(
+ self, database_id: str = "(default)", project_id: str =
PROVIDE_PROJECT_ID
+ ) -> list[dict[str, Any]]:
+ """
+ List long-running operations on the specified Firestore database.
+
+ :param database_id: The Database ID.
+ :param project_id: Optional, Google Cloud Project project_id where the
database belongs.
+ If set to None or missing, the default project_id from the Google
Cloud connection is used.
+ """
+ service = self.get_conn()
+ name = f"projects/{project_id}/databases/{database_id}"
+ operations_resource = service.projects().databases().operations()
+ all_operations: list[dict[str, Any]] = []
+ page_token: str | None = None
+
+ while True:
+ list_kwargs: dict[str, Any] = {"name": name}
+ if page_token:
+ list_kwargs["pageToken"] = page_token
+ response =
operations_resource.list(**list_kwargs).execute(num_retries=self.num_retries)
+ if not isinstance(response, dict):
+ break
+ operations = response.get("operations", [])
+ if isinstance(operations, list):
+ all_operations.extend(operations)
+ next_page_token = response.get("nextPageToken")
+ if not isinstance(next_page_token, str) or not next_page_token:
+ break
+ page_token = next_page_token
+
+ return all_operations
+
+ @GoogleBaseHook.fallback_to_default_project_id
+ def find_matching_export_operation(
+ self,
+ body: dict[str, Any],
+ database_id: str = "(default)",
+ project_id: str = PROVIDE_PROJECT_ID,
+ include_completed: bool = True,
+ ) -> dict[str, Any] | None:
+ """
+ Find an existing running or completed export operation matching the
export request body.
+
+ :param body: The export request body.
+ :param database_id: The Database ID.
+ :param project_id: Optional, Google Cloud Project project_id where the
database belongs.
+ :param include_completed: Whether to match already completed
operations in addition to
+ in-progress operations.
+ """
+ target_uri = body.get("outputUriPrefix")
+ if not target_uri or not isinstance(target_uri, str):
+ return None
+ target_collections = sorted(body.get("collectionIds") or [])
+ target_namespaces = sorted(body.get("namespaceIds") or [])
+
+ completed_match: dict[str, Any] | None = None
+ for op in self.list_operations(database_id=database_id,
project_id=project_id):
+ if not isinstance(op, dict):
+ continue
+ metadata = op.get("metadata")
+ if not isinstance(metadata, dict):
+ metadata = {}
+
+ meta_type = metadata.get("@type", "")
+ if meta_type and not meta_type.endswith("ExportDocumentsMetadata"):
+ continue
+
+ op_state = metadata.get("operationState", "")
+ if op.get("error") or op_state in {"FAILED", "CANCELLED",
"CANCELLING"}:
+ continue
+
+ op_collections = sorted(metadata.get("collectionIds") or [])
+ if target_collections != op_collections:
+ continue
+
+ op_namespaces = sorted(metadata.get("namespaceIds") or [])
+ if target_namespaces != op_namespaces:
+ continue
+
+ response_dict = op.get("response")
+ op_uri = metadata.get("outputUriPrefix") or (
+ response_dict.get("outputUriPrefix") if
isinstance(response_dict, dict) else None
+ )
+ if not isinstance(op_uri, str) or op_uri.rstrip("/") !=
target_uri.rstrip("/"):
Review Comment:
Agreed, I added a warning and early return when `outputUriPrefix` is a
bucket-only URI without an object path prefix, and documented this requirement
in the `reattach_on_restart` docstrings
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
