kaxil commented on code in PR #74388:
URL: https://github.com/apache/airflow/pull/74388#discussion_r4394129718
##########
providers/common/ai/src/airflow/providers/common/ai/operators/llm.py:
##########
@@ -359,9 +360,6 @@ def _create_agent(self, *, output_type: Any, instructions:
str) -> Agent[object,
)
def execute(self, context: Context) -> Any:
- if self._may_review:
Review Comment:
Can we keep this early call, plus the ones in `llm_branch.py`, `llm_sql.py`
and `llm_schema_compare.py`? I missed these when I suggested the no-op hook.
Without them an override that rejects the prompt only runs once `_run_llm` has
been paid for and the pending decision pushed, and under `decision_policy` only
if a review opens.
##########
providers/common/ai/tests/unit/common/ai/operators/test_llm.py:
##########
@@ -1647,25 +1633,44 @@ def
test_execute_complete_returns_an_unedited_output_that_does_not_validate_back
@pytest.mark.skipif(
not AIRFLOW_V_3_1_PLUS, reason="Human in the loop is only compatible with
Airflow >= 3.1.0"
)
-class TestLLMOperatorMultimodalPromptGuard:
- """LLMOperator.execute raises before agent.run_sync when require_approval
is True
- and self.prompt is not a string -- covering direct-operator construction
and the
- native template rendering escape (where a string template renders to a
Sequence)."""
-
+class TestLLMOperatorMultimodalPromptApproval:
+ @pytest.mark.parametrize(
+ "review_kwargs",
+ [
+ pytest.param({"require_approval": True}, id="require-approval"),
+ pytest.param({"decision_policy":
DecisionPolicy(min_confidence=0.7)}, id="below-threshold"),
+ ],
+ )
+ @patch("airflow.providers.standard.triggers.hitl.HITLTrigger",
autospec=True)
+ @patch("airflow.sdk.execution_time.hitl.upsert_hitl_detail")
@patch("airflow.providers.common.ai.operators.llm.PydanticAIHook",
autospec=True)
- def test_execute_rejects_sequence_prompt_with_require_approval(self,
mock_hook_cls):
- mock_agent = MagicMock(spec=["run_sync"])
+ def test_execute_with_approval_reviews_a_sequence_prompt(
+ self, mock_hook_cls, mock_upsert, mock_trigger_cls,
make_mock_run_result, review_kwargs
+ ):
+ result = make_mock_run_result(Assessment(category="billing",
score=0.3))
+ result.response = ModelResponse(
Review Comment:
Nit: `TestLLMOperatorConfidenceGate._result` already builds this
`ModelResponse`. Moving it to module level, or putting the below-threshold case
in that class, would save the copy.
##########
providers/common/ai/tests/unit/common/ai/operators/test_llm_schema_compare.py:
##########
@@ -827,13 +827,35 @@ def
test_execute_complete_rejects_invalid_modified_output(self, modified):
with pytest.raises(ReviewedOutputValidationError, match="output_type
SchemaCompareResult"):
op.execute_complete({}, generated_output=result.model_dump_json(),
event=event)
- def test_execute_rejects_sequence_prompt_with_require_approval(self):
+ @mock.patch(
+
"airflow.providers.common.ai.operators.llm_schema_compare.LLMSchemaCompareOperator._build_schema_context"
+ )
+ @mock.patch(
+
"airflow.providers.common.ai.operators.llm_schema_compare.LLMSchemaCompareOperator._build_system_prompt"
+ )
+ @mock.patch("airflow.providers.standard.triggers.hitl.HITLTrigger",
autospec=True)
+ @mock.patch("airflow.sdk.execution_time.hitl.upsert_hitl_detail")
+ def test_execute_with_approval_reviews_a_sequence_prompt(
+ self,
+ mock_upsert,
+ mock_trigger_cls,
+ mock_build_system_prompt,
+ mock_build_schema_context,
+ make_mock_run_result,
+ ):
+ mock_build_schema_context.return_value = "schema_context"
+ mock_build_system_prompt.return_value = "system_prompt"
op = LLMSchemaCompareOperator(
- task_id="test_task",
- prompt=["describe", b"bytes"],
- llm_conn_id="llm_conn",
+ **{**_BASE_KWARGS, "prompt": ["Compare",
BinaryContent(data=b"\x89PNG", media_type="image/png")]},
**self._APPROVAL_KWARGS,
)
+ mock_agent = mock.Mock()
Review Comment:
Nit: could the agent and `op.llm_hook` mocks here get a spec? The decorator
test added in this PR already uses `MagicMock(spec=["run_sync",
"instrument"])`, and `mock.Mock(spec=PydanticAIHook)` would do for the hook.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
