Lee-W commented on code in PR #76801:
URL: https://github.com/apache/airflow/pull/76801#discussion_r4615767509


##########
providers/common/ai/src/airflow/providers/common/ai/durable/capability.py:
##########
@@ -207,6 +209,7 @@ def _lacks_id(leaf: AbstractToolset[Any]) -> 
TypeGuard[FunctionToolset[Any] | Dy
 # Why a recorded step that matched is run live anyway: it was written by a 
version whose
 # types have changed since, so replaying it would fail on every retry.
 _UNLOADABLE = "the recorded result no longer loads, so it was written by 
another version"
+_UNSTAMPED_ANSWERER = "the recorded response does not say which fallback chain 
member answered it"

Review Comment:
   Done. Thanks!



##########
providers/common/ai/tests/unit/common/ai/durable/test_capability.py:
##########
@@ -1928,3 +1938,230 @@ async def 
test_a_retry_whose_budget_is_spent_can_still_replay(self, memory_stora
 
         assert result.output == "done"
         assert usage.requests == 3
+
+
+class TestFallbackChainEdits:
+    """Editing a fallback chain between attempts keeps the steps a surviving 
member answered."""
+
+    @staticmethod
+    def build_chain(
+        calls: Calls,
+        members: list[tuple[str, float | None]],
+        *,
+        up: set[str],
+        stamped: bool = True,
+    ) -> FallbackModel:
+        def build_member(name: str, temperature: float | None) -> Model:
+            def respond(messages: list[ModelMessage], info: AgentInfo) -> 
ModelResponse:
+                calls.bump(name)
+                if name not in up:
+                    raise ModelHTTPError(500, name)
+                return ModelResponse(parts=[TextPart(f"{name} answered")])
+
+            model = FunctionModel(
+                respond,
+                model_name=name,
+                settings=ModelSettings(temperature=temperature) if temperature 
is not None else None,
+            )
+            return MemberStampingModel(model) if stamped else model
+
+        return FallbackModel(
+            *(build_member(name, temperature) for name, temperature in 
members),
+            fallback_on=(ModelAPIError,),
+        )
+
+    @staticmethod
+    def build_agent(chain: FallbackModel) -> Agent[None, str]:
+        return Agent(chain, name="chained", capabilities=[AirflowDurability()])
+
+    @pytest.mark.parametrize(
+        ("members", "up", "output", "replayed", "calls_by_member"),
+        [
+            pytest.param(
+                [("c", None), ("b", None)], {"c"}, "b answered", 1, {"b": 1, 
"c": 0}, id="member added before"
+            ),
+            pytest.param(
+                [("b", None), ("a", None)], {"a"}, "b answered", 1, {"b": 1, 
"a": 1}, id="members reordered"
+            ),
+            pytest.param(
+                [("a", None), ("b", 0.9)], set(), "b answered", 1, {"b": 1}, 
id="member settings changed"
+            ),
+            pytest.param(
+                [("a", None)], {"a"}, "a answered", 0, {"a": 2, "b": 1}, 
id="answering member removed"
+            ),
+            pytest.param(
+                [("a", None), ("b2", None)],
+                {"b2"},
+                "b2 answered",
+                0,
+                {"a": 2, "b": 1, "b2": 1},
+                id="answering member's model changed",
+            ),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def 
test_a_step_replays_only_while_its_answering_member_is_in_the_chain(
+        self, memory_storage, members, up, output, replayed, calls_by_member
+    ):
+        calls = Calls()
+        first = self.build_chain(calls, [("a", None), ("b", None)], up={"b"})
+        await attempt(memory_storage, self.build_agent(first))
+
+        journal = DurableJournal(memory_storage)
+        with (
+            journal_scope(journal),
+            mock.patch.object(DurableRun, "reject", autospec=True, 
side_effect=DurableRun.reject) as reject,
+        ):
+            result = await self.build_agent(self.build_chain(calls, members, 
up=up)).run("go")
+
+        assert result.output == output
+        assert journal.stats.replayed["model"] == replayed
+        assert {name: calls[name] for name in calls_by_member} == 
calls_by_member
+        reason = "the model that answered this step, function:b, is no longer 
in the fallback chain"
+        assert [call.kwargs["reason"] for call in reject.call_args_list] == (
+            [reason] if replayed == 0 else []
+        )
+
+    @pytest.mark.parametrize(
+        ("second_attempt", "replayed", "calls_by_member"),
+        [
+            pytest.param([("a", None), ("b", None)], 1, {"a": 1, "b": 1}, 
id="chain unchanged"),
+            pytest.param([("b", None)], 0, {"a": 1, "b": 2}, id="chain 
edited"),

Review Comment:
   Done. Thanks!



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to