kaxil commented on code in PR #65313:
URL: https://github.com/apache/airflow/pull/65313#discussion_r3097422182


##########
airflow-core/tests/unit/dag_processing/test_manager.py:
##########
@@ -831,6 +832,109 @@ def test_kill_timed_out_processors_no_kill(self):
             manager._kill_timed_out_processors()
         mock_kill.assert_not_called()
 
+    def test_handle_parsing_result_keeps_prior_stats_when_persist_fails(self, 
session):
+        """Persist errors are logged and swallowed so one file does not abort 
the batch."""
+        manager = DagFileProcessorManager(max_runs=1)
+        file = DagFileInfo(bundle_name="testing", rel_path=Path("abc.txt"), 
bundle_path=TEST_DAGS_FOLDER)
+        original_last_finish_time = timezone.utcnow() - timedelta(hours=2)
+        original_stat = DagFileStat(
+            num_dags=1,
+            import_errors=0,
+            last_finish_time=original_last_finish_time,
+            last_duration=1.0,
+            run_count=3,
+            last_num_of_db_queries=0,
+        )
+        manager._file_stats[file] = original_stat
+        manager._bundle_versions["testing"] = "v1"
+        manager._file_process_interval = 60
+
+        processor, _ = self.mock_processor(start_time=time.monotonic() - 1)
+        processor.had_callbacks = False
+        processor.parsing_result = DagFileParsingResult(fileloc="abc.txt", 
serialized_dags=[])
+
+        with mock.patch.object(manager, "persist_parsing_result", 
side_effect=RuntimeError("boom")):
+            manager.handle_parsing_result(file, processor, session=session)
+
+        assert manager._file_stats[file] is original_stat
+        assert manager._file_stats[file].run_count == 3
+        assert manager._file_stats[file].last_finish_time == 
original_last_finish_time
+        assert manager.processed_recently(timezone.utcnow(), file) is False
+
+    def 
test_handle_parsing_result_updates_stats_after_successful_persist(self, 
session):
+        manager = DagFileProcessorManager(max_runs=1)
+        file = DagFileInfo(bundle_name="testing", rel_path=Path("abc.txt"), 
bundle_path=TEST_DAGS_FOLDER)
+        original_stat = DagFileStat(
+            num_dags=1,
+            import_errors=0,
+            last_finish_time=timezone.utcnow() - timedelta(hours=2),
+            last_duration=1.0,
+            run_count=3,
+            last_num_of_db_queries=0,
+        )
+        manager._file_stats[file] = original_stat
+        manager._bundle_versions["testing"] = "v1"
+
+        processor, _ = self.mock_processor(start_time=time.monotonic() - 1)
+        processor.had_callbacks = False
+        processor.parsing_result = DagFileParsingResult(fileloc="abc.txt", 
serialized_dags=[])
+
+        with mock.patch.object(manager, "persist_parsing_result") as 
mock_persist:
+            manager.handle_parsing_result(file, processor, session=session)
+
+        mock_persist.assert_called_once_with(
+            bundle_name="testing",
+            bundle_version="v1",
+            parsing_result=processor.parsing_result,
+            run_duration=mock.ANY,
+            relative_fileloc="abc.txt",
+            session=session,
+        )
+        assert manager._file_stats[file] is not original_stat
+        assert manager._file_stats[file].run_count == 4
+        assert manager._file_stats[file].last_finish_time is not None
+        assert manager._file_stats[file].last_finish_time > 
original_stat.last_finish_time
+        assert manager._file_stats[file].num_dags == 0
+
+    def 
test_collect_results_processes_remaining_files_when_one_persist_fails(self, 
session):
+        manager = DagFileProcessorManager(max_runs=1)
+        file_a = DagFileInfo(bundle_name="testing", rel_path=Path("a.py"), 
bundle_path=TEST_DAGS_FOLDER)
+        file_b = DagFileInfo(bundle_name="testing", rel_path=Path("b.py"), 
bundle_path=TEST_DAGS_FOLDER)
+        base_stat = DagFileStat(
+            num_dags=0,
+            import_errors=0,
+            last_finish_time=timezone.utcnow() - timedelta(hours=1),
+            last_duration=1.0,
+            run_count=1,
+            last_num_of_db_queries=0,
+        )
+        manager._file_stats[file_a] = base_stat
+        manager._file_stats[file_b] = copy.copy(base_stat)
+        manager._bundle_versions["testing"] = "v1"
+
+        proc_a, _ = self.mock_processor(start_time=time.monotonic() - 1)
+        proc_a.had_callbacks = False
+        proc_a.parsing_result = DagFileParsingResult(fileloc="a.py", 
serialized_dags=[])
+        proc_b, _ = self.mock_processor(start_time=time.monotonic() - 1)
+        proc_b.had_callbacks = False
+        proc_b.parsing_result = DagFileParsingResult(fileloc="b.py", 
serialized_dags=[])
+
+        manager._processors = {file_a: proc_a, file_b: proc_b}
+        stat_a_before = manager._file_stats[file_a]
+        stat_b_before = manager._file_stats[file_b]
+
+        with mock.patch.object(
+            manager,
+            "persist_parsing_result",
+            side_effect=[RuntimeError("boom"), None],

Review Comment:
   This mocks `persist_parsing_result` entirely, so it only verifies the 
try/except plumbing around it -- not that the batch actually completes 
successfully after a real persist failure. If the point of this change is "one 
file's failure shouldn't stall others", a test that lets the real method run 
against a poisoned session (e.g., by raising `IntegrityError` inside 
`update_dag_parsing_results_in_db`) would exercise the interesting part.



##########
airflow-core/src/airflow/dag_processing/manager.py:
##########
@@ -982,6 +982,97 @@ def terminate_orphan_processes(self, present: 
set[DagFileInfo]):
                 processor.logger_filehandle.close()
                 self._file_stats.pop(file, None)
 
+    def handle_parsing_result(
+        self, file: DagFileInfo, proc: DagFileProcessorProcess, *, session: 
Session
+    ) -> None:
+        """
+        Post-process a single finished parse result.
+
+        Detects callback-only processing, updates file stats, emits metrics,
+        and persists DAGs/import-errors via :meth:`persist_parsing_result`.
+        Extracted from ``_collect_results`` to keep result handling and
+        persistence separate.
+
+        If persistence fails, the error is logged and this method returns 
without
+        updating file stats for this file so other files in the same
+        ``_collect_results`` cycle still run.
+        """
+        is_callback_only = proc.had_callbacks and proc.parsing_result is None
+        if is_callback_only:
+            self.log.debug("Detected callback-only processing for %s", file)
+
+        run_duration = time.monotonic() - proc.start_time
+        next_stat = process_parse_results(
+            run_duration=run_duration,
+            finish_time=timezone.utcnow(),
+            run_count=self._file_stats[file].run_count,
+            bundle_name=file.bundle_name,
+            bundle_version=self._bundle_versions[file.bundle_name],
+            parsing_result=proc.parsing_result,
+            is_callback_only=is_callback_only,
+            relative_fileloc=str(file.rel_path),
+        )
+
+        if proc.parsing_result is not None:
+            try:
+                self.persist_parsing_result(
+                    bundle_name=file.bundle_name,
+                    bundle_version=self._bundle_versions[file.bundle_name],
+                    parsing_result=proc.parsing_result,
+                    run_duration=run_duration,
+                    relative_fileloc=str(file.rel_path),
+                    session=session,
+                )
+            except Exception:
+                self.log.exception(
+                    "Failed to persist parsing result for %s in bundle %s; "
+                    "skipping stats update for this file. Other files in this 
cycle are still processed.",
+                    str(file.rel_path),
+                    file.bundle_name,
+                )
+                return
+
+        self._file_stats[file] = next_stat

Review Comment:
   For a file that has never parsed successfully, `_file_stats[file]` is the 
default `DagFileStat()` with `last_finish_time=None`. If persist fails on the 
first parse, `processed_recently` returns False next loop, so the file gets 
re-queued immediately. With a sustained persist failure (DB wedged, future API 
down), this becomes a tight re-parse loop for all new files. Worth considering 
whether failed files should get at least a minimal `last_finish_time` stamp to 
throttle retries.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to