MaksYermak commented on code in PR #72809:
URL: https://github.com/apache/airflow/pull/72809#discussion_r4456455719
##########
providers/apache/beam/src/airflow/providers/apache/beam/hooks/beam.py:
##########
@@ -176,13 +176,22 @@ def run_beam_command(
stderr=subprocess.PIPE,
close_fds=True,
)
+
+ if is_dataflow_job_id_exist_callback and hasattr(os, "set_blocking") and
proc.stdout and proc.stderr:
+ os.set_blocking(proc.stdout.fileno(), False)
+ os.set_blocking(proc.stderr.fileno(), False)
Review Comment:
@potiuk I have added this line
##########
providers/apache/beam/src/airflow/providers/apache/beam/operators/beam.py:
##########
@@ -155,8 +155,20 @@ def set_current_dataflow_job_id(job_id):
on_new_job_id_callback=set_current_dataflow_job_id
)
- def __is_dataflow_job_id_exist_callback(self) -> Callable[[], bool]:
- def is_dataflow_job_id_exist() -> bool:
+ def __is_dataflow_job_id_exist_callback(self) -> Callable[..., bool]:
+ def is_dataflow_job_id_exist(checking_by_job_name: bool = False) ->
bool:
+ if checking_by_job_name and self.dataflow_hook:
+ try:
+ self.dataflow_job_id =
self.dataflow_hook.get_job_id_of_running_job(
+ job_name=self.dataflow_job_name, # type: ignore
+ project_id=self.dataflow_config.project_id,
+ location=self.dataflow_config.location,
+ )
+ except Exception:
+ self.log.warning( # type: ignore
+ "Failed to get Dataflow job ID by job name '%s'",
+ self.dataflow_job_name, # type: ignore
+ )
Review Comment:
@potiuk I have updated the code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
