mattcasters commented on code in PR #8514: URL: https://github.com/apache/hop/pull/8514#discussion_r4083631993
########## plugins/tech/ai/src/main/java/org/apache/hop/ai/engine/AiChatModelFactory.java: ########## @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.ai.engine; + +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.model.ollama.OllamaChatModel; +import dev.langchain4j.model.openai.OpenAiChatModel; +import org.apache.hop.ai.metadata.AiModelRole; +import org.apache.hop.ai.metadata.AiProvider; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.metadata.api.IHopMetadataProvider; + +/** + * Builds a langchain4j chat model from an {@link AiProvider}. + * + * <p>{@link AiChatFactory} converts a provider into the Language Model Chat transform's own + * metadata, which is what that transform needs. A transform that wants to drive the model itself, + * to constrain it with a schema or to run a tool loop, needs the model object instead, and that is + * what this hands back. + */ +public final class AiChatModelFactory { + + private AiChatModelFactory() {} + + /** + * Loads the named provider and builds its chat model. + * + * @param providerName the {@code AiProvider} to use + * @param modelName a model that overrides the provider's, or empty to use the provider's own + * @param variables used to resolve the provider's fields + * @param metadataProvider where the provider is loaded from + * @throws HopException when the provider is missing, incomplete, or serves no chat model + */ + public static ChatModel createChatModel( + String providerName, + String modelName, + IVariables variables, + IHopMetadataProvider metadataProvider) + throws HopException { + AiProvider provider = AiProviderLoader.load(providerName, metadataProvider); + return createChatModel(provider, modelName, variables); + } + + public static ChatModel createChatModel( + AiProvider provider, String modelName, IVariables variables) throws HopException { + // of() rejects a null or half configured provider, so there is one guard, not two. + AiProviderSettings settings = AiProviderSettings.of(provider, variables); + + String model = Utils.isEmpty(modelName) ? chatModelName(provider, variables) : modelName; + if (Utils.isEmpty(model)) { + model = settings.backend().getDefaultModelName(); + } + if (Utils.isEmpty(model)) { + throw new HopException( + "No chat model is configured. Set one on this transform, or on AI provider '" + + provider.getName() + + "'."); + } + + return switch (settings.type()) { Review Comment: **[bug]** `supportedCapabilities()` is not a live probe of the provider. In langchain4j 1.20 both `OllamaChatModel` and `OpenAiChatModel` return only the set passed to the builder, and an unset set is empty. `StructuredExtract.supportsJsonSchema()` therefore always takes the prompt-only branch, and the schema built in `openModel()` is never attached as `responseFormat`. The integration test comment says Ollama constrains `qwen2.5:0.5b` to the schema; with this factory it does not, so enum and optional-null behavior depend on an unconstrained small model. `additionalProperties(false)` is also dropped on the wire unless OpenAI `strictJsonSchema` is true, which this builder never sets. **Suggestion:** Call `.supportedCapabilities(Capability.RESPONSE_FORMAT_JSON_SCHEMA)` on both builders, and `.strictJsonSchema(true)` on the OpenAI builder. Add a test that a request built the way `StructuredExtract.ask` builds it carries a JSON-schema response format. ########## plugins/tech/ai/src/main/java/org/apache/hop/ai/transforms/structuredextract/ExtractionParser.java: ########## @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.ai.transforms.structuredextract; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.math.BigDecimal; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.List; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.json.HopJson; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.util.Utils; + +/** + * Reads the model's answer back into typed Hop values. + * + * <p>Nothing here guesses. A value that will not coerce is reported with the field that caused it, + * so the row can go to an error hop naming the problem. Silently writing null would be worse than + * failing: downstream, an absent value and a misread value look identical, and only one of them is + * the model's honest answer. + */ +public final class ExtractionParser { + + private ExtractionParser() {} + + /** + * @param json the model's answer, possibly wrapped in a markdown fence + * @param fields the grid rows, in output order + * @return one value per field, in the same order, with null for anything the model omitted + * @throws HopException when the answer is not an object, or a value will not coerce + */ + public static Object[] parse(String json, List<StructuredExtractField> fields) + throws HopException { + JsonNode root = readTree(json); + Object[] values = new Object[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + StructuredExtractField field = fields.get(i); + JsonNode node = root.get(field.getName()); + values[i] = node == null || node.isNull() ? null : coerce(node, field); + } + return values; + } + + /** + * Models wrap JSON in ```json fences even when told not to, so the fence is stripped rather than + * treated as a failure. Anything else is a genuine protocol error. + */ + static JsonNode readTree(String json) throws HopException { + String text = json == null ? "" : json.trim(); + if (text.startsWith("```")) { + int firstNewline = text.indexOf('\n'); + int lastFence = text.lastIndexOf("```"); + if (firstNewline > 0 && lastFence > firstNewline) { + text = text.substring(firstNewline + 1, lastFence).trim(); + } + } + if (text.isEmpty()) { + throw new HopException("The model returned nothing to read fields from"); + } + try { + // Without this, Jackson parses every decimal through a double and 1.10 arrives as 1.1. + // BigNumber is the type people choose precisely when that matters. + ObjectMapper mapper = + HopJson.newMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + JsonNode root = mapper.readTree(text); + if (root == null || !root.isObject()) { + throw new HopException( + "Expected a JSON object from the model but got: " + abbreviate(text)); + } + return root; + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("The model did not return readable JSON: " + abbreviate(text), e); + } + } + + private static Object coerce(JsonNode node, StructuredExtractField field) throws HopException { + int type = ExtractionSchema.typeOf(field); + String text = node.isValueNode() ? node.asText() : node.toString(); + if (Utils.isEmpty(text)) { + return null; + } + try { + return switch (type) { + case IValueMeta.TYPE_INTEGER -> Long.valueOf(text.trim()); + case IValueMeta.TYPE_NUMBER -> Double.valueOf(text.trim()); + case IValueMeta.TYPE_BIGNUMBER -> + node.isNumber() ? node.decimalValue() : new BigDecimal(text.trim()); + case IValueMeta.TYPE_BOOLEAN -> toBoolean(text.trim(), field); + case IValueMeta.TYPE_DATE, IValueMeta.TYPE_TIMESTAMP -> toDate(text.trim(), field); Review Comment: **[bug]** `TYPE_TIMESTAMP` is coerced through `toDate`, which returns `java.util.Date`. `ValueMetaTimestamp.getTimestamp` casts the native value to `java.sql.Timestamp`, so the first preview, clone, or `getString` throws `ClassCastException` downstream of this transform's error hop. `SimpleDateFormat.parse(String)` also accepts a `yyyy-MM-dd` prefix and ignores the rest, so `2026-03-01T10:30:00` becomes midnight and `2026-03-01 oops` is treated as a valid date. `ExtractionSchema` asks for both Date and Timestamp as `yyyy-MM-dd` only, so a Timestamp column cannot carry a time even when the model returns one. **Suggestion:** Return `new Timestamp(parsed.getTime())` for `TYPE_TIMESTAMP`, parse with a `ParsePosition` that requires the whole string, try a date-time pattern before the date-only one, and describe Timestamp fields to the model as a date-time rather than `yyyy-MM-dd`. ########## plugins/tech/ai/src/main/java/org/apache/hop/ai/transforms/structuredextract/ExtractionParser.java: ########## @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.ai.transforms.structuredextract; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.math.BigDecimal; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.List; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.json.HopJson; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.util.Utils; + +/** + * Reads the model's answer back into typed Hop values. + * + * <p>Nothing here guesses. A value that will not coerce is reported with the field that caused it, + * so the row can go to an error hop naming the problem. Silently writing null would be worse than + * failing: downstream, an absent value and a misread value look identical, and only one of them is + * the model's honest answer. + */ +public final class ExtractionParser { + + private ExtractionParser() {} + + /** + * @param json the model's answer, possibly wrapped in a markdown fence + * @param fields the grid rows, in output order + * @return one value per field, in the same order, with null for anything the model omitted + * @throws HopException when the answer is not an object, or a value will not coerce + */ + public static Object[] parse(String json, List<StructuredExtractField> fields) + throws HopException { + JsonNode root = readTree(json); + Object[] values = new Object[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + StructuredExtractField field = fields.get(i); + JsonNode node = root.get(field.getName()); Review Comment: **[bug]** The schema, the prompt, and `getFields` all key the field by `trimmedName()`, but the answer is looked up with `field.getName()`. A grid value with surrounding whitespace is requested as `severity` and read back as ` severity`, so the output column is always null. That is the silent null this transform is written to avoid, and it is not reported as a conversion error. **Suggestion:** Look up `field.trimmedName()`. Trim the name in `StructuredExtractDialog.readFields` as well, so stored metadata matches the schema key. ########## plugins/tech/ai/src/main/java/org/apache/hop/ai/transforms/structuredextract/ExtractionParser.java: ########## @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.ai.transforms.structuredextract; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.math.BigDecimal; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.List; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.json.HopJson; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.util.Utils; + +/** + * Reads the model's answer back into typed Hop values. + * + * <p>Nothing here guesses. A value that will not coerce is reported with the field that caused it, + * so the row can go to an error hop naming the problem. Silently writing null would be worse than + * failing: downstream, an absent value and a misread value look identical, and only one of them is + * the model's honest answer. + */ +public final class ExtractionParser { + + private ExtractionParser() {} + + /** + * @param json the model's answer, possibly wrapped in a markdown fence + * @param fields the grid rows, in output order + * @return one value per field, in the same order, with null for anything the model omitted + * @throws HopException when the answer is not an object, or a value will not coerce + */ + public static Object[] parse(String json, List<StructuredExtractField> fields) + throws HopException { + JsonNode root = readTree(json); + Object[] values = new Object[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + StructuredExtractField field = fields.get(i); + JsonNode node = root.get(field.getName()); + values[i] = node == null || node.isNull() ? null : coerce(node, field); Review Comment: **[suggestion]** Allowed values become a JSON enum, but `parse` never checks them. On the prompt-only path, and on any provider that does not enforce the schema, `"critical"` is written into a field constrained to `low,medium,high` and the row looks successful. **Suggestion:** After coercion, if `ExtractionSchema.allowedValues(field)` is non-empty and the returned text is not in that list, throw a `HopException` that names the field, the value, and the allowed list so the error hop can divert the row. ########## plugins/tech/ai/src/main/java/org/apache/hop/ai/transforms/structuredextract/StructuredExtract.java: ########## @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.ai.transforms.structuredextract; + +import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.model.chat.Capability; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.request.ResponseFormat; +import dev.langchain4j.model.chat.request.ResponseFormatType; +import dev.langchain4j.model.chat.request.json.JsonSchema; +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.ai.engine.AiChatModelFactory; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.RowDataUtil; +import org.apache.hop.core.util.Utils; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +/** Pulls named, typed fields out of a text field using a language model. */ +public class StructuredExtract extends BaseTransform<StructuredExtractMeta, StructuredExtractData> { + + private static final Class<?> PKG = StructuredExtractMeta.class; + + public StructuredExtract( + TransformMeta transformMeta, + StructuredExtractMeta meta, + StructuredExtractData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + @Override + public boolean init() { + if (Utils.isEmpty(meta.getAiProvider())) { + logError(BaseMessages.getString(PKG, "StructuredExtract.Validation.ProviderRequired")); + return false; + } + data.fields = usableFields(); + if (data.fields.isEmpty()) { + logError(BaseMessages.getString(PKG, "StructuredExtract.Validation.NoFields")); + return false; + } + return super.init(); + } + + private List<StructuredExtractField> usableFields() { + List<StructuredExtractField> usable = new ArrayList<>(); + for (StructuredExtractField field : meta.getFields()) { + if (field != null && !field.trimmedName().isEmpty()) { + usable.add(field); + } + } + return usable; + } + + @Override + public boolean processRow() throws HopException { + Object[] row = getRow(); + if (row == null) { + setOutputDone(); + return false; + } + + if (first) { + first = false; + data.inputRowMeta = getInputRowMeta(); + data.outputRowMeta = data.inputRowMeta.clone(); + meta.getFields( + data.outputRowMeta, getTransformName(), null, null, this, getMetadataProvider()); + resolveInputField(); + openModel(); + } + + try { + putRow(data.outputRowMeta, extract(row)); + } catch (HopException e) { + if (!getTransformMeta().isDoingErrorHandling()) { + throw e; + } + putError( + data.inputRowMeta, row, 1, e.getMessage(), meta.getInputField(), "STRUCTUREDEXTRACT001"); + } + return true; + } + + private Object[] extract(Object[] row) throws HopException { + String text = data.inputRowMeta.getString(row, data.inputFieldIndex); + Object[] output = RowDataUtil.createResizedCopy(row, data.outputRowMeta.size()); + if (Utils.isEmpty(text)) { + // Nothing to read fields from. The row keeps its place with the new fields left empty, + // rather than being dropped or sent to the error hop: an empty input is not a failure. + return output; + } + + String answer = ask(text); + Object[] values = ExtractionParser.parse(answer, data.fields); + int index = data.inputRowMeta.size(); + for (Object value : values) { + output[index++] = value; + } + return output; + } + + private String ask(String text) throws HopException { + ChatRequest.Builder request = + ChatRequest.builder() + .messages(SystemMessage.from(data.systemPrompt), UserMessage.from(text)); + if (data.responseFormat != null) { + request.responseFormat(data.responseFormat); + } + try { + return data.model.chat(request.build()).aiMessage().text(); + } catch (Exception e) { + throw new HopException(BaseMessages.getString(PKG, "StructuredExtract.Error.Calling"), e); + } + } + + /** + * The instruction sent with every row. + * + * <p>The schema is described here even when the model is constrained by a real one: the field + * descriptions are the user's own words about what each field means, and a model that can see + * them extracts noticeably better than one working from field names alone. + */ + static String buildSystemPrompt(String schemaDescription, String instructions) { + StringBuilder prompt = new StringBuilder(); + prompt + .append("Read the fields below out of the text the user sends. ") + .append("Answer with a single JSON object and nothing else. ") + .append("Use null for anything the text does not say. Do not invent values.\n\n") + .append(schemaDescription); + if (!Utils.isEmpty(instructions)) { + prompt.append("\n\n").append(instructions); + } + return prompt.toString(); + } + + private void resolveInputField() throws HopException { + data.inputFieldIndex = data.inputRowMeta.indexOfValue(meta.getInputField()); + if (data.inputFieldIndex < 0) { + throw new HopException( + BaseMessages.getString( + PKG, + "StructuredExtract.Validation.InputFieldNotFound", + String.valueOf(meta.getInputField()))); + } + } + + private void openModel() throws HopException { + data.model = + AiChatModelFactory.createChatModel( + resolve(meta.getAiProvider()), + resolve(meta.getModelName()), + this, + getMetadataProvider()); + + JsonSchema schema = ExtractionSchema.build(data.fields, getTransformName()); Review Comment: **[bug]** The schema name sent to the provider is the transform name. OpenAI requires `response_format.json_schema.name` to be a-z, A-Z, 0-9, underscores or dashes, at most 64 characters, and rejects anything else with HTTP 400. The sample pipeline and the integration test both name this transform "Structured extract", so the first OpenAI row fails as soon as the schema is actually attached. Ollama only forwards the root element and ignores the name, which hides it. **Suggestion:** Pass a sanitized token (replace characters outside `[A-Za-z0-9_-]`, truncate to 64, and fall back to `"extraction"` when nothing remains), not `getTransformName()`. ########## plugins/tech/ai/src/main/java/org/apache/hop/ai/transforms/structuredextract/StructuredExtractMeta.java: ########## @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.ai.transforms.structuredextract; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.ai.metadata.AiProvider; +import org.apache.hop.core.CheckResult; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.gui.plugin.GuiElementType; +import org.apache.hop.core.gui.plugin.GuiPlugin; +import org.apache.hop.core.gui.plugin.GuiWidgetElement; +import org.apache.hop.core.gui.plugin.GuiWidgetGroupType; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaFactory; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; + +@Getter +@Setter +@Transform( + id = "StructuredExtract", + image = "structuredextract.svg", + name = "i18n::StructuredExtract.Name", + description = "i18n::StructuredExtract.Description", + categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.AI", + keywords = "ai,extract,structured,json,schema,llm,classify,entities", + documentationUrl = "/pipeline/transforms/structuredextract.html", + classLoaderGroup = "hop-ai") +@GuiPlugin(classLoaderGroup = "hop-ai") +public class StructuredExtractMeta + extends BaseTransformMeta<StructuredExtract, StructuredExtractData> { + + public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = "STRUCTURED_EXTRACT_DIALOG_OPTIONS"; + public static final String WIDGET_INPUT_FIELD = "STRUCTURED_EXTRACT_INPUT_FIELD"; + + private static final String TAB_MAIN = "i18n::StructuredExtract.Tab.Main"; + private static final String TAB_MAIN_ORDER = "0100"; + + private static final Class<?> PKG = StructuredExtractMeta.class; + + @GuiWidgetElement( + order = "0100", + type = GuiElementType.METADATA, + metadata = AiProvider.class, + label = "i18n::StructuredExtract.aiProvider.Label", + toolTip = "i18n::StructuredExtract.aiProvider.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = TAB_MAIN, + groupOrder = TAB_MAIN_ORDER) + @HopMetadataProperty( + key = "ai_provider", + injectionKey = "AI_PROVIDER", + injectionKeyDescription = "StructuredExtractMeta.Injection.AI_PROVIDER") + private String aiProvider; + + @GuiWidgetElement( + order = "0200", + type = GuiElementType.TEXT, + label = "i18n::StructuredExtract.modelName.Label", + toolTip = "i18n::StructuredExtract.modelName.Tooltip", + variables = true, + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = TAB_MAIN, + groupOrder = TAB_MAIN_ORDER) + @HopMetadataProperty( + key = "model_name", + injectionKey = "MODEL_NAME", + injectionKeyDescription = "StructuredExtractMeta.Injection.MODEL_NAME") + private String modelName = ""; + + @GuiWidgetElement( + id = WIDGET_INPUT_FIELD, + order = "0300", + type = GuiElementType.COMBO, + label = "i18n::StructuredExtract.inputField.Label", + toolTip = "i18n::StructuredExtract.inputField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = TAB_MAIN, + groupOrder = TAB_MAIN_ORDER) + @HopMetadataProperty( + key = "input_field", + injectionKey = "INPUT_FIELD", + injectionKeyDescription = "StructuredExtractMeta.Injection.INPUT_FIELD") + private String inputField; + + @GuiWidgetElement( + order = "0400", + type = GuiElementType.MULTI_LINE_TEXT, + multiLineTextHeight = 4, + label = "i18n::StructuredExtract.instructions.Label", + toolTip = "i18n::StructuredExtract.instructions.Tooltip", + variables = true, + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = TAB_MAIN, + groupOrder = TAB_MAIN_ORDER) + @HopMetadataProperty( + key = "instructions", + injectionKey = "INSTRUCTIONS", + injectionKeyDescription = "StructuredExtractMeta.Injection.INSTRUCTIONS") + private String instructions = ""; + + /** + * The fields to pull out. This is the heart of the transform: it becomes the schema the model is + * constrained by, the columns added to the stream, and the types the answer is read into. + */ + @HopMetadataProperty( + groupKey = "fields", + key = "field", + injectionGroupKey = "FIELDS", + injectionGroupDescription = "StructuredExtractMeta.Injection.FIELDS") + private List<StructuredExtractField> fields = new ArrayList<>(); + + @Override + public void setDefault() { + aiProvider = ""; + modelName = ""; + inputField = ""; + instructions = ""; + fields = new ArrayList<>(); + } + + @Override + public void getFields( + IRowMeta row, + String origin, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) + throws HopTransformException { + for (StructuredExtractField field : fields) { + if (field == null || field.trimmedName().isEmpty()) { + continue; + } + try { + // StructuredExtract writes one value per field in this same order, so the two have to + // agree on which fields exist and what type each one is. + IValueMeta value = + ValueMetaFactory.createValueMeta(field.trimmedName(), ExtractionSchema.typeOf(field)); + value.setOrigin(origin); + row.addValueMeta(value); + } catch (HopException e) { + throw new HopTransformException("Unable to add output field '" + field.getName() + "'", e); + } + } + } + + @Override + public boolean supportsErrorHandling() { + return true; + } + + @Override + public void check( + List<ICheckResult> remarks, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + String[] input, + String[] output, + IRowMeta info, + IVariables variables, + IHopMetadataProvider metadataProvider) { + + if (Utils.isEmpty(aiProvider)) { + error(remarks, transformMeta, "StructuredExtract.Validation.ProviderRequired"); + } + if (Utils.isEmpty(inputField)) { + error(remarks, transformMeta, "StructuredExtract.Validation.InputFieldRequired"); + } else if (prev != null && prev.indexOfValue(inputField) < 0) { + error(remarks, transformMeta, "StructuredExtract.Validation.InputFieldNotFound", inputField); + } + + List<StructuredExtractField> named = + fields.stream().filter(f -> f != null && !f.trimmedName().isEmpty()).toList(); + if (named.isEmpty()) { + error(remarks, transformMeta, "StructuredExtract.Validation.NoFields"); + return; + } + + Set<String> seen = new HashSet<>(); + for (StructuredExtractField field : named) { + String name = field.trimmedName(); + if (!seen.add(name)) { Review Comment: **[suggestion]** Duplicate detection uses a case-sensitive `HashSet`, while `IRowMeta.indexOfValue` and `addValueMeta` are case-insensitive. `Total` and `total` both pass `check`, then `getFields` silently renames the second column (for example `total_1`) and the extracted values no longer sit under the names that were configured. **Suggestion:** Compare trimmed names with `equalsIgnoreCase` here and in `ExtractionSchema.build`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
