This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-8630-02efe7ac037c037eeca1922cd1e944b89f274816 in repository https://gitbox.apache.org/repos/asf/texera.git
commit 8dbf049bd814fde00c0ce1407a1f31c7de03c645 Author: Prateek Ganigi <[email protected]> AuthorDate: Thu Sep 24 03:28:41 2026 +0000 feat(workflow-operator): carry candidate labels into the image chat message (#8630) ### What changes were proposed in this PR? `zero-shot-image-classification` puts its labels in the payload as `parameters.candidate_labels`, which only hf-inference understands. When the operator falls back to a third-party chat provider, the request is rebuilt as an image part plus a text part, and that text was just `prompt_value`, so the provider received an image with no labels at all, and answered with a generic caption instead of a classification, silently. `_chat_content_for_task` now has a branch for the task that turns the labels into an instruction ("Classify the image into exactly one of these labels: …"), and the image chat branches use it for their text part instead of the bare prompt. When no labels are configured it returns `prompt_value`, so nothing else changes. The branch is deliberately separate from the existing `zero-shot-classification` one rather than merged into it: there `inputs` is the text being classified, but here `inputs` is the base64 image, so reusing that path would have inlined the whole image into the prompt. A test covers that specifically. Four call sites now route through the helper — the three chat branches in `_call_provider` plus the model-author branch in `_post_with_fallback`. The fourth is a no-op for its own tasks, but it makes the invariant uniform and testable. This is Part B of #7906 and completes it; Part A (#7920) fixed the response side. ### Any related issues? Closes #7906 ### How was this PR tested? Written test-first. The three new tests in `HuggingFaceCodegenBaseSpec` failed against the current template, then passed once the helper branch and the call sites were in place. The full Hugging Face suite is at 146 tests, `PythonCodeRawInvalidTextSpec` py-compiles the generated Python for all 117 operators, and `scalafmtCheck` and `scalafix --check` are clean. The emitted helper was also run directly, comparing this branch against main: the labels now reach the prompt, the base64 image never appears in it, empty or missing labels still fall back to the prompt value, and `zero-shot-classification`, `question-answering`, `visual-question-answering` and `text-generation` are unchanged. ### Was this PR authored or co-authored using generative AI tooling? Yes, this PR was co-authored with Claude in compliance with ASF policy. --- .../codegen/HuggingFaceCodegenBase.scala | 21 +++++++++--- .../codegen/HuggingFaceCodegenBaseSpec.scala | 38 ++++++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBase.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBase.scala index a40d4aeac7..1db86f5ab1 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBase.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBase.scala @@ -230,7 +230,7 @@ object HuggingFaceCodegenBase { | "role": "user", | "content": [ | {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}} if img_b64 else None, - | {"type": "text", "text": prompt_value if prompt_value else "What is in this image?"}, + | {"type": "text", "text": self._chat_content_for_task(pipeline_payload, prompt_value) or "What is in this image?"}, | ], | }], | } @@ -291,6 +291,19 @@ object HuggingFaceCodegenBase { | f"Table:\n{json.dumps(table)}\n\nQuestion: {query}" | ) | return query or prompt_value + | if task == "zero-shot-image-classification": + | # Kept separate from zero-shot-classification below: there `inputs` + | # is the text being classified, here it is the base64 image, which + | # must never be spliced into the prompt. The image travels as its + | # own content part; this is only the instruction beside it. + | params = pipeline_payload.get("parameters") if isinstance(pipeline_payload, dict) else None + | labels = params.get("candidate_labels", []) if isinstance(params, dict) else [] + | if labels: + | return ( + | "Classify the image into exactly one of these labels: " + | f"{', '.join(str(l) for l in labels)}. Respond with only the chosen label." + | ) + | return prompt_value | if task == "zero-shot-classification": | params = pipeline_payload.get("parameters") if isinstance(pipeline_payload, dict) else None | labels = params.get("candidate_labels", []) if isinstance(params, dict) else [] @@ -368,7 +381,7 @@ object HuggingFaceCodegenBase { | if img_b64: | messages = [{"role": "user", "content": [ | {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}, - | {"type": "text", "text": prompt_value if prompt_value else "What is in this image?"}, + | {"type": "text", "text": self._chat_content_for_task(pipeline_payload, prompt_value) or "What is in this image?"}, | ]}] | return requests.post(url, headers=zai_headers, json={"model": provider_id, "messages": messages}, timeout=120) | @@ -510,7 +523,7 @@ object HuggingFaceCodegenBase { | if img_b64: | messages = [{"role": "user", "content": [ | {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}, - | {"type": "text", "text": prompt_value if prompt_value else "What is in this image?"}, + | {"type": "text", "text": self._chat_content_for_task(pipeline_payload, prompt_value) or "What is in this image?"}, | ]}] | return requests.post( | url, @@ -531,7 +544,7 @@ object HuggingFaceCodegenBase { | if img_b64: | messages = [{"role": "user", "content": [ | {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}, - | {"type": "text", "text": prompt_value if prompt_value else "Describe this image."}, + | {"type": "text", "text": self._chat_content_for_task(pipeline_payload, prompt_value) or "Describe this image."}, | ]}] | resp2 = requests.post( | url, diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBaseSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBaseSpec.scala index 0a5e96f330..9acae7db85 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBaseSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBaseSpec.scala @@ -144,4 +144,42 @@ class HuggingFaceCodegenBaseSpec extends AnyFlatSpec with Matchers { out should not include "MARKER_TASK_zXyq42" out should not include "MARKER_SYSTEM_zXyq42" } + + // #7906 Part B: zero-shot-image-classification puts its labels in + // parameters.candidate_labels, but the chat branches rebuild the request as + // [image_url, text=prompt_value] and dropped them, so a chat provider got an + // unlabelled image and returned a caption instead of a classification. + it should "carry the candidate labels into the chat message for zero-shot-image-classification" in { + val helper = chatContentHelper(HuggingFaceCodegenBase.render(makeCtx(), StubCodegen)) + helper should include("""if task == "zero-shot-image-classification":""") + helper should include("candidate_labels") + helper should include("Classify the image into exactly one of these labels:") + } + + it should "route every image chat text part through the task-aware helper" in { + // Every image chat branch builds a two-part content list — the three in + // _call_provider (zai-org, OpenAI-compatible, unknown-provider fallback) and + // the model-author one in _post_with_fallback. The text part must be the + // reformulated prompt, not the bare prompt_value. + val out = HuggingFaceCodegenBase.render(makeCtx(), StubCodegen) + out.split("""\{"type": "text", "text": self\._chat_content_for_task\(""").length - 1 shouldBe 4 + out should not include """{"type": "text", "text": prompt_value if prompt_value else""" + } + + it should "not splice the base64 image into the zero-shot-image-classification prompt" in { + // For this task `inputs` is the image itself, unlike zero-shot-classification + // where it is the text being classified. Reusing that branch would inline the + // whole base64 blob into the chat message. + val branch = chatContentHelper(HuggingFaceCodegenBase.render(makeCtx(), StubCodegen)) + .split("""if task == "zero-shot-image-classification":""")(1) + .split(" if task ==")(0) + branch should not include "Text: {text}" + branch should not include "inputs if isinstance(inputs, str)" + } + + /** The emitted body of _chat_content_for_task, so assertions can't be satisfied + * by unrelated parts of the template (e.g. the pre-loop label validation). + */ + private def chatContentHelper(rendered: String): String = + rendered.split("def _chat_content_for_task")(1).split(" def ")(0) }
