kahlstrm commented on code in PR #39496:
URL: https://github.com/apache/airflow/pull/39496#discussion_r1594431729


##########
tests/utils/test_log_handlers.py:
##########
@@ -317,43 +317,52 @@ def 
test__read_for_celery_executor_fallbacks_to_worker(self, create_task_instanc
         """Test for executors which do not have `get_task_log` method, it 
fallbacks to reading
         log from worker if and only if remote logs aren't found"""
         executor_name = "CeleryExecutor"
-
-        ti = create_task_instance(
-            dag_id="dag_for_testing_celery_executor_log_read",
-            task_id="task_for_testing_celery_executor_log_read",
-            run_type=DagRunType.SCHEDULED,
-            execution_date=DEFAULT_DATE,
-        )
-        ti.state = TaskInstanceState.RUNNING
-        ti.try_number = 2
-        with conf_vars({("core", "executor"): executor_name}):
-            reload(executor_loader)
-            fth = FileTaskHandler("")
-
-            fth._read_from_logs_server = mock.Mock()
-            fth._read_from_logs_server.return_value = ["this message"], 
["this\nlog\ncontent"]
-            actual = fth._read(ti=ti, try_number=2)
-            fth._read_from_logs_server.assert_called_once()
-            assert actual == ("*** this message\nthis\nlog\ncontent", 
{"end_of_log": False, "log_pos": 16})
-
-            # Previous try_number should return served logs when remote logs 
aren't implemented
-            fth._read_from_logs_server = mock.Mock()
-            fth._read_from_logs_server.return_value = ["served logs 
try_number=1"], ["this\nlog\ncontent"]
-            actual = fth._read(ti=ti, try_number=1)
-            fth._read_from_logs_server.assert_called_once()
-            assert actual == (
-                "*** served logs try_number=1\nthis\nlog\ncontent",
-                {"end_of_log": True, "log_pos": 16},
+        # Reading logs from worker should occur when the task is either 
running, deferred, or up for retry.
+        for state in (TaskInstanceState.RUNNING, TaskInstanceState.DEFERRED, 
TaskInstanceState.UP_FOR_RETRY):
+            ti = create_task_instance(
+                dag_id=f"dag_for_testing_celery_executor_log_read_{state}",
+                task_id="task_for_testing_celery_executor_log_read",
+                run_type=DagRunType.SCHEDULED,
+                execution_date=DEFAULT_DATE,
             )
-
-            # When remote_logs is implemented, previous try_number is from 
remote logs without reaching worker server
-            fth._read_from_logs_server.reset_mock()
-            fth._read_remote_logs = mock.Mock()
-            fth._read_remote_logs.return_value = ["remote logs"], 
["remote\nlog\ncontent"]
-            actual = fth._read(ti=ti, try_number=1)
-            fth._read_remote_logs.assert_called_once()
-            fth._read_from_logs_server.assert_not_called()
-            assert actual == ("*** remote logs\nremote\nlog\ncontent", 
{"end_of_log": True, "log_pos": 18})
+            ti.try_number = 2
+            ti.state = state
+            with conf_vars({("core", "executor"): executor_name}):
+                reload(executor_loader)
+                fth = FileTaskHandler("")
+                fth._read_from_logs_server = mock.Mock()
+                fth._read_from_logs_server.return_value = ["this message"], 
["this\nlog\ncontent"]
+                actual = fth._read(ti=ti, try_number=2)
+                fth._read_from_logs_server.assert_called_once()
+                # If we are in the up for retry or deferred state, log has 
ended and a new task try has started.
+                # When the state is running, the log has not ended.
+                expected_end_of_log = state in (TaskInstanceState.DEFERRED, 
TaskInstanceState.UP_FOR_RETRY)

Review Comment:
   I.e. the tests the way they were previously or even after these changes 
don't fail with the following change to logic
   
   ```sh
   diff --git a/airflow/utils/log/file_task_handler.py 
b/airflow/utils/log/file_task_handler.py
   index 6f1c02e931..bd5cbe9127 100644
   --- a/airflow/utils/log/file_task_handler.py
   +++ b/airflow/utils/log/file_task_handler.py
   @@ -407,7 +407,7 @@ class FileTaskHandler(logging.Handler):
            )
            log_pos = len(logs)
            messages = "".join([f"*** {x}\n" for x in messages_list])
   -        end_of_log = ti.try_number != try_number or not 
is_in_running_or_deferred
   +        end_of_log = ti.try_number != try_number or ti.state != 
TaskInstanceState.RUNNING
            if metadata and "log_pos" in metadata:
                previous_chars = metadata["log_pos"]
                logs = logs[previous_chars:]  # Cut off previously passed log 
test as new tail
   
   ```
   This caused some headache while debugging.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to