mattcasters commented on code in PR #8419: URL: https://github.com/apache/hop/pull/8419#discussion_r4028625350
########## plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunker.java: ########## @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.chunker; + +import java.util.List; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.RowDataUtil; +import org.apache.hop.core.util.Utils; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategy; +import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyFactory; +import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType; +import org.apache.hop.pipeline.transforms.chunker.chunking.StructureChunkingStrategy; +import org.apache.hop.pipeline.transforms.chunker.document.ContentType; +import org.apache.hop.pipeline.transforms.chunker.document.ContentTypeResolver; + +/** + * Text Chunker transform - splits text into chunks for AI/ML processing. Supports multiple chunking + * strategies (character-based with word boundary respect, paragraph-based, structure-aware) with + * configurable overlap. + */ +public class TextChunker extends BaseTransform<TextChunkerMeta, TextChunkerData> { + + private static final Class<?> PKG = TextChunkerMeta.class; + + /** The chunking strategy instance. */ + private ChunkingStrategy strategy; + + /** + * Counter used to synthesise a document ID when no source field is configured. Prefixed with the + * transform copy number so parallel copies cannot produce colliding identifiers. + */ + private long documentIdCounter = 0; + + public TextChunker( + TransformMeta transformMeta, + TextChunkerMeta meta, + TextChunkerData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + @Override + public boolean init() { + if (!super.init()) { + return false; + } + strategy = ChunkingStrategyFactory.createStrategy(meta.getChunkingStrategy()); + logBasic( + BaseMessages.getString( + PKG, "TextChunker.Log.Initialized", String.valueOf(meta.getChunkingStrategy()))); + return true; + } + + @Override + public boolean processRow() throws HopException { + Object[] row = getRow(); + + if (row == null) { + setOutputDone(); + return false; + } + + if (first) { + first = false; + data.inputRowMeta = getInputRowMeta(); + data.outputRowMeta = data.inputRowMeta.clone(); + meta.getFields( + data.outputRowMeta, getTransformName(), null, null, this, getMetadataProvider()); + resolveFieldIndices(); + } + + try { + chunkRow(row); + } catch (HopException e) { + if (getTransformMeta().isDoingErrorHandling()) { Review Comment: **[bug]** `processRow` routes failures to `putError` when error handling is enabled, but `TextChunkerMeta` does not override `supportsErrorHandling()`. `BaseTransformMeta` defaults that to `false`, so Hop Gui never offers an error hop and this branch is unreachable. **Suggestion:** Return `true` from `TextChunkerMeta.supportsErrorHandling()`. ########## plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMeta.java: ########## @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.chunker; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.CheckResult; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopPluginException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType; +import org.apache.hop.pipeline.transforms.chunker.document.ContentType; + +@Getter +@Setter +@Transform( + id = "TextChunker", + image = "chunker.svg", + name = "i18n::TextChunker.Name", + description = "i18n::TextChunker.Description", + categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform", + documentationUrl = "/pipeline/transforms/textchunker.html", + keywords = "i18n::TextChunker.Keywords") +public class TextChunkerMeta extends BaseTransformMeta<TextChunker, TextChunkerData> { + + private static final Class<?> PKG = TextChunkerMeta.class; + + /** The field containing text to chunk. */ + @HopMetadataProperty(key = "inputField", injectionKey = "INPUT_FIELD") + private String inputField; + + /** The name of the field to output chunks to. */ + @HopMetadataProperty(key = "outputChunkField", injectionKey = "OUTPUT_CHUNK_FIELD") + private String outputChunkField = "chunk_text"; + + /** The chunking strategy to use. */ + @HopMetadataProperty(key = "chunkingStrategy", injectionKey = "CHUNKING_STRATEGY") + private ChunkingStrategyType chunkingStrategy = ChunkingStrategyType.CHARACTER; + + /** + * The maximum size for each chunk (characters for CHARACTER strategy, approximate for PARAGRAPH). + */ + @HopMetadataProperty(key = "chunkSize", injectionKey = "CHUNK_SIZE") + private int chunkSize = 1000; Review Comment: **[bug]** Chunk size and overlap are `int` fields, while the dialog uses `LabelTextVar` and `Const.toInt(...)` on OK. A value like `${CHUNK_SIZE}` is not an integer, so it silently becomes 1000 (overlap 0) and is never resolved at runtime. **Suggestion:** Store both as strings, resolve with `variables.resolve(...)` in `processRow`/`check()`, then parse. ########## plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/CharacterChunkingStrategy.java: ########## @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.chunker.chunking; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.pipeline.transforms.chunker.Chunk; + +/** + * Chunking strategy that splits text on fixed character count while respecting word boundaries. + * This ensures that chunks don't split words in the middle. + */ +public class CharacterChunkingStrategy implements ChunkingStrategy { + + /** Characters that are considered word separators. */ + private static final String WORD_SEPARATORS = " \t\n\r\f"; + + @Override + public List<Chunk> chunk(String text, int maxSize, int overlap) { + List<Chunk> chunks = new ArrayList<>(); + + if (text == null || text.isEmpty() || maxSize <= 0) { + return chunks; + } + + if (overlap < 0) { + chunks.add(new Chunk(text, 0, 0, text.length())); + return chunks; + } + + overlap = Math.min(overlap, maxSize - 1); + + int start = 0; + int chunkIndex = 0; + int textLength = text.length(); + + while (start < textLength) { + int end = Math.min(start + maxSize, textLength); + + if (end == textLength) { + String chunkContent = text.substring(start, end); + chunks.add(new Chunk(chunkContent, chunkIndex++, start, end)); + break; + } + + int separatorPos = -1; + for (int i = end - 1; i >= start; i--) { + if (WORD_SEPARATORS.indexOf(text.charAt(i)) >= 0) { + separatorPos = i; + break; + } + } + + int chunkEnd; + boolean splitOnWordBoundary; + if (separatorPos >= start) { + chunkEnd = separatorPos + 1; + splitOnWordBoundary = true; + } else { + chunkEnd = end; + splitOnWordBoundary = false; + } + + if (chunkEnd <= start) { + chunkEnd = Math.min(start + maxSize, textLength); + } + + String chunkContent = text.substring(start, chunkEnd); + chunks.add(new Chunk(chunkContent, chunkIndex++, start, chunkEnd)); + + if (chunkEnd >= textLength) { + break; + } + + int nextStart; + if (splitOnWordBoundary && overlap > 0) { Review Comment: **[bug]** Overlap is applied only when the chunk ended on an ASCII whitespace boundary (space, tab, newline, CR, form-feed). CJK text, long tokens, and URLs therefore get abutting chunks with no overlap even when overlap is configured. The `nextStart <= start` guard already prevents zero-progress loops. **Suggestion:** Always compute `nextStart = Math.max(start + 1, chunkEnd - overlap)` (still cap overlap at `maxSize - 1`). Treat Unicode whitespace as a boundary if word-respecting splits remain a goal. ########## plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupport.java: ########## @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.chunker.document.hopxml; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +/** DOM helpers for Hop .hpl / .hwf XML. */ +public final class HopXmlSupport { + + static final Set<String> SKIP_TAGS = Review Comment: **[bug]** `SKIP_TAGS` lists GUI/layout noise but not `password`, `key`, `token`, or similar. `serializeConfig` therefore copies transform/action secrets from `.hpl`/`.hwf` into structure chunks for embeddings. **Suggestion:** Add secret-bearing tag names to `SKIP_TAGS` (case-insensitive) or replace their text with a redaction marker. ########## plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/metadata/HopMetadataJsonParser.java: ########## @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.chunker.document.metadata; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.apache.hop.core.json.HopJson; +import org.apache.hop.core.util.JsonUtil; +import org.apache.hop.pipeline.transforms.chunker.document.ContentType; +import org.apache.hop.pipeline.transforms.chunker.document.DocumentNode; +import org.apache.hop.pipeline.transforms.chunker.document.DocumentParser; + +/** + * Parses Hop project metadata JSON into one section per connection, engine, or top-level config + * block. + */ +public final class HopMetadataJsonParser implements DocumentParser { + + @Override + public ContentType getContentType() { + return ContentType.METADATA; + } + + @Override + public DocumentNode parse(String text) { + if (text == null || text.isBlank()) { + return DocumentNode.root(""); + } + + JsonNode root = parseRoot(text); + if (root == null || !root.isObject()) { + return DocumentNode.root(text); + } + + List<DocumentNode> children = new ArrayList<>(); + String name = textValue(root.get("name")); + String description = textValue(root.get("description")); + + StringBuilder overview = new StringBuilder(); + if (!name.isEmpty()) { + overview.append("Hop metadata '").append(name).append("'\n"); + } + if (!description.isEmpty()) { + overview.append("Description: ").append(description).append('\n'); + } + if (overview.length() > 0) { + children.add(DocumentNode.leaf("Overview", overview.toString().strip(), 0)); + } + + Iterator<Map.Entry<String, JsonNode>> fields = root.fields(); + while (fields.hasNext()) { + Map.Entry<String, JsonNode> entry = fields.next(); + String key = entry.getKey(); + if ("name".equals(key) || "description".equals(key)) { + continue; + } + addSectionsForKey(children, key, entry.getValue()); + } + + if (children.isEmpty()) { + return DocumentNode.root(text); + } + return new DocumentNode("", "", 0, children); + } + + private static void addSectionsForKey(List<DocumentNode> children, String key, JsonNode value) { + if (value == null || value.isNull()) { + return; + } + if ("rdbms".equals(key) && value.isObject()) { + value + .fields() + .forEachRemaining( + conn -> + children.add( + DocumentNode.leaf( + "Connection: " + conn.getKey(), formatConnection(conn.getValue()), 0))); + return; + } + if ("engineRunConfiguration".equals(key) && value.isObject()) { + value + .fields() + .forEachRemaining( + engine -> + children.add( + DocumentNode.leaf( + "Engine: " + engine.getKey(), prettyJson(engine.getValue()), 0))); + return; + } + if (value.isObject() && value.size() > 1 && allValuesAreObjects(value)) { + value + .fields() + .forEachRemaining( + child -> + children.add( + DocumentNode.leaf( + humanizeKey(key) + ": " + child.getKey(), + prettyJson(child.getValue()), + 0))); + return; + } + children.add(DocumentNode.leaf(humanizeKey(key), prettyJson(value), 0)); + } + + private static boolean allValuesAreObjects(JsonNode object) { + Iterator<JsonNode> values = object.elements(); + while (values.hasNext()) { + if (!values.next().isObject()) { + return false; + } + } + return true; + } + + private static String formatConnection(JsonNode conn) { + if (conn == null || !conn.isObject()) { + return prettyJson(conn); + } + StringBuilder body = new StringBuilder(); + appendLine(body, "Plugin", textValue(conn.get("pluginName"))); + appendLine(body, "Host", textValue(conn.get("hostname"))); + appendLine(body, "Port", textValue(conn.get("port"))); + appendLine(body, "Database", textValue(conn.get("databaseName"))); + appendLine(body, "Username", textValue(conn.get("username"))); + appendLine(body, "Manual URL", textValue(conn.get("manualUrl"))); + String pretty = prettyJson(conn); Review Comment: **[bug]** `formatConnection` pretty-prints the entire RDBMS object. Hop connection metadata includes `@HopMetadataProperty(password = true)` fields (`password`, SSH secrets). Those values become chunk text destined for embedding models. **Suggestion:** Redact password/token/secret/passphrase/privateKey keys (and `Encrypted …` values) before serializing. Do not dump the raw node after listing host/database. ########## plugins/transforms/textchunker/src/main/resources/org/apache/hop/pipeline/transforms/chunker/messages/messages_en_US.properties: ########## @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +TextChunkerDialog.Shell.Title=Text chunker +TextChunkerDialog.GetFields.Error.Title=Error getting fields +TextChunkerDialog.GetFields.Error.Message=Unable to read fields from the previous transform + +TextChunker.Name=Text chunker +TextChunker.Description=Split text into chunks for AI/ML processing without breaking words or sentences +TextChunker.Keywords=AI,LLM,text,chunk,split,paragraph,character,word,embedding + +TextChunker.inputField.Label=Input field +TextChunker.inputField.Tooltip=Select the field containing text to chunk + +TextChunker.sourceDocumentIdField.Label=Source document ID field +TextChunker.sourceDocumentIdField.Tooltip=Optional input field to use as the document identifier in metadata. When empty, a row counter is used. + +TextChunker.outputChunkField.Label=Output chunk field +TextChunker.outputChunkField.Tooltip=Name of the field to output chunks to + +TextChunker.chunkingStrategy.Label=Chunking strategy +TextChunker.chunkingStrategy.Tooltip=Choose how to split text: Character, Paragraph, or Structure (heading-aware with breadcrumb prefixes) + +TextChunker.contentType.Label=Content type +TextChunker.contentType.Tooltip=Document format for Structure strategy. Auto detects from source_type field or text heuristics. + +TextChunker.contentTypeField.Label=Content type field +TextChunker.contentTypeField.Tooltip=Optional input field (e.g. source_type) to select parser: doc=AsciiDoc, pipeline/workflow=Hop XML, article/blog=Plain Review Comment: **[nit]** Tooltip says `article/blog=Plain`. `ContentTypeResolver` maps `article` and `blog` to `MARKDOWN`. **Suggestion:** Align the tooltip with the resolver (`article/blog=Markdown`). ########## plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerDialog.java: ########## @@ -0,0 +1,395 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.chunker; + +import org.apache.hop.core.Const; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType; +import org.apache.hop.pipeline.transforms.chunker.document.ContentType; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.widget.ComboVar; +import org.apache.hop.ui.core.widget.LabelTextVar; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CCombo; +import org.eclipse.swt.events.FocusAdapter; +import org.eclipse.swt.events.FocusEvent; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Shell; + +public class TextChunkerDialog extends BaseTransformDialog { + + private static final Class<?> PKG = TextChunkerMeta.class; + + private final TextChunkerMeta input; + + private ComboVar wInputField; + private ComboVar wSourceDocumentIdField; + private LabelTextVar wOutputChunkField; + private CCombo wChunkingStrategy; + private CCombo wContentType; + private ComboVar wContentTypeField; + private LabelTextVar wChunkSize; + private LabelTextVar wChunkOverlap; + private Button wIncludeMetadata; + private LabelTextVar wChunkIndexField; + private LabelTextVar wChunkStartPosField; + private LabelTextVar wDocumentIdField; + private LabelTextVar wChunkCountField; + + public TextChunkerDialog( + Shell parent, + IVariables variables, + TextChunkerMeta transformMeta, + PipelineMeta pipelineMeta) { + super(parent, variables, transformMeta, pipelineMeta); + input = transformMeta; + } + + @Override + public String open() { + Control lastControl = createShell(BaseMessages.getString(PKG, "TextChunkerDialog.Shell.Title")); + buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build(); + + wInputField = new ComboVar(variables, shell, SWT.BORDER | SWT.READ_ONLY); + lastControl = addFieldCombo(lastControl, "TextChunker.inputField", wInputField); + + wSourceDocumentIdField = new ComboVar(variables, shell, SWT.BORDER | SWT.READ_ONLY); + lastControl = + addFieldCombo(lastControl, "TextChunker.sourceDocumentIdField", wSourceDocumentIdField); + + wOutputChunkField = + new LabelTextVar( + variables, + shell, + BaseMessages.getString(PKG, "TextChunker.outputChunkField.Label"), + BaseMessages.getString(PKG, "TextChunker.outputChunkField.Tooltip")); + PropsUi.setLook(wOutputChunkField); + wOutputChunkField.addModifyListener(lsMod); + FormData fdOutput = new FormData(); + fdOutput.left = new FormAttachment(0, 0); + fdOutput.top = new FormAttachment(lastControl, margin); + fdOutput.right = new FormAttachment(100, 0); + wOutputChunkField.setLayoutData(fdOutput); + lastControl = wOutputChunkField; + + Label wlStrategy = new Label(shell, SWT.RIGHT); + wlStrategy.setText(BaseMessages.getString(PKG, "TextChunker.chunkingStrategy.Label")); + PropsUi.setLook(wlStrategy); + FormData fdlStrategy = new FormData(); + fdlStrategy.left = new FormAttachment(0, 0); + fdlStrategy.right = new FormAttachment(middle, -margin); + fdlStrategy.top = new FormAttachment(lastControl, margin); + wlStrategy.setLayoutData(fdlStrategy); + wChunkingStrategy = new CCombo(shell, SWT.BORDER | SWT.READ_ONLY); + PropsUi.setLook(wChunkingStrategy); + wChunkingStrategy.setItems( + new String[] { + ChunkingStrategyType.CHARACTER.name(), + ChunkingStrategyType.PARAGRAPH.name(), + ChunkingStrategyType.STRUCTURE.name() + }); + FormData fdStrategy = new FormData(); + fdStrategy.left = new FormAttachment(middle, 0); + fdStrategy.top = new FormAttachment(lastControl, margin); + fdStrategy.right = new FormAttachment(100, 0); + wChunkingStrategy.setLayoutData(fdStrategy); + wChunkingStrategy.addModifyListener(lsMod); + wChunkingStrategy.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + input.setChanged(); + updateStructureFieldsEnabled(); + } + }); + lastControl = wChunkingStrategy; + + Label wlContentType = new Label(shell, SWT.RIGHT); + wlContentType.setText(BaseMessages.getString(PKG, "TextChunker.contentType.Label")); + wlContentType.setToolTipText(BaseMessages.getString(PKG, "TextChunker.contentType.Tooltip")); + PropsUi.setLook(wlContentType); + FormData fdlContentType = new FormData(); + fdlContentType.left = new FormAttachment(0, 0); + fdlContentType.right = new FormAttachment(middle, -margin); + fdlContentType.top = new FormAttachment(lastControl, margin); + wlContentType.setLayoutData(fdlContentType); + wContentType = new CCombo(shell, SWT.BORDER | SWT.READ_ONLY); + PropsUi.setLook(wContentType); + wContentType.setItems( + new String[] { + ContentType.AUTO.name(), + ContentType.PLAIN.name(), + ContentType.MARKDOWN.name(), + ContentType.ASCIIDOC.name(), + ContentType.PIPELINE.name(), + ContentType.WORKFLOW.name(), + ContentType.METADATA.name() + }); + FormData fdContentType = new FormData(); + fdContentType.left = new FormAttachment(middle, 0); + fdContentType.top = new FormAttachment(lastControl, margin); + fdContentType.right = new FormAttachment(100, 0); + wContentType.setLayoutData(fdContentType); + wContentType.addModifyListener(lsMod); + lastControl = wContentType; + + wContentTypeField = new ComboVar(variables, shell, SWT.BORDER | SWT.READ_ONLY); + lastControl = addFieldCombo(lastControl, "TextChunker.contentTypeField", wContentTypeField); + + wChunkSize = + new LabelTextVar( + variables, + shell, + BaseMessages.getString(PKG, "TextChunker.chunkSize.Label"), + BaseMessages.getString(PKG, "TextChunker.chunkSize.Tooltip")); + PropsUi.setLook(wChunkSize); + wChunkSize.addModifyListener(lsMod); + FormData fdSize = new FormData(); + fdSize.left = new FormAttachment(0, 0); + fdSize.top = new FormAttachment(lastControl, margin); + fdSize.right = new FormAttachment(100, 0); + wChunkSize.setLayoutData(fdSize); + lastControl = wChunkSize; + + wChunkOverlap = + new LabelTextVar( + variables, + shell, + BaseMessages.getString(PKG, "TextChunker.chunkOverlap.Label"), + BaseMessages.getString(PKG, "TextChunker.chunkOverlap.Tooltip")); + PropsUi.setLook(wChunkOverlap); + wChunkOverlap.addModifyListener(lsMod); + FormData fdOverlap = new FormData(); + fdOverlap.left = new FormAttachment(0, 0); + fdOverlap.top = new FormAttachment(lastControl, margin); + fdOverlap.right = new FormAttachment(100, 0); + wChunkOverlap.setLayoutData(fdOverlap); + lastControl = wChunkOverlap; + + wIncludeMetadata = new Button(shell, SWT.CHECK); + wIncludeMetadata.setText(BaseMessages.getString(PKG, "TextChunker.includeMetadata.Label")); + wIncludeMetadata.setToolTipText( + BaseMessages.getString(PKG, "TextChunker.includeMetadata.Tooltip")); + PropsUi.setLook(wIncludeMetadata); + FormData fdMeta = new FormData(); + fdMeta.left = new FormAttachment(middle, 0); + fdMeta.top = new FormAttachment(lastControl, margin); + fdMeta.right = new FormAttachment(100, 0); + wIncludeMetadata.setLayoutData(fdMeta); + wIncludeMetadata.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + input.setChanged(); + updateMetadataEnabled(); + } + }); + lastControl = wIncludeMetadata; + + wChunkIndexField = + new LabelTextVar( + variables, + shell, + BaseMessages.getString(PKG, "TextChunker.chunkIndexField.Label"), + BaseMessages.getString(PKG, "TextChunker.chunkIndexField.Tooltip")); + PropsUi.setLook(wChunkIndexField); + wChunkIndexField.addModifyListener(lsMod); + FormData fdIndex = new FormData(); + fdIndex.left = new FormAttachment(0, 0); + fdIndex.top = new FormAttachment(lastControl, margin); + fdIndex.right = new FormAttachment(100, 0); + wChunkIndexField.setLayoutData(fdIndex); + + wChunkStartPosField = + new LabelTextVar( + variables, + shell, + BaseMessages.getString(PKG, "TextChunker.chunkStartPosField.Label"), + BaseMessages.getString(PKG, "TextChunker.chunkStartPosField.Tooltip")); + PropsUi.setLook(wChunkStartPosField); + wChunkStartPosField.addModifyListener(lsMod); + FormData fdStart = new FormData(); + fdStart.left = new FormAttachment(0, 0); + fdStart.top = new FormAttachment(wChunkIndexField, margin); + fdStart.right = new FormAttachment(100, 0); + wChunkStartPosField.setLayoutData(fdStart); + + wDocumentIdField = + new LabelTextVar( + variables, + shell, + BaseMessages.getString(PKG, "TextChunker.documentIdField.Label"), + BaseMessages.getString(PKG, "TextChunker.documentIdField.Tooltip")); + PropsUi.setLook(wDocumentIdField); + wDocumentIdField.addModifyListener(lsMod); + FormData fdDoc = new FormData(); + fdDoc.left = new FormAttachment(0, 0); + fdDoc.top = new FormAttachment(wChunkStartPosField, margin); + fdDoc.right = new FormAttachment(100, 0); + wDocumentIdField.setLayoutData(fdDoc); + + wChunkCountField = + new LabelTextVar( + variables, + shell, + BaseMessages.getString(PKG, "TextChunker.chunkCountField.Label"), + BaseMessages.getString(PKG, "TextChunker.chunkCountField.Tooltip")); + PropsUi.setLook(wChunkCountField); + wChunkCountField.addModifyListener(lsMod); + FormData fdCount = new FormData(); + fdCount.left = new FormAttachment(0, 0); + fdCount.top = new FormAttachment(wDocumentIdField, margin); + fdCount.right = new FormAttachment(100, 0); + wChunkCountField.setLayoutData(fdCount); + fdCount.bottom = new FormAttachment(wOk, -margin * 2); + wChunkCountField.setLayoutData(fdCount); + + getData(); + updateMetadataEnabled(); + updateStructureFieldsEnabled(); + loading = false; + input.setChanged(changed); + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + return transformName; + } + + private Control addFieldCombo(Control previous, String labelKey, ComboVar combo) { + Label label = new Label(shell, SWT.RIGHT); + label.setText(BaseMessages.getString(PKG, labelKey + ".Label")); + label.setToolTipText(BaseMessages.getString(PKG, labelKey + ".Tooltip")); + PropsUi.setLook(label); + FormData fdl = new FormData(); + fdl.left = new FormAttachment(0, 0); + fdl.right = new FormAttachment(middle, -margin); + fdl.top = new FormAttachment(previous, margin); + label.setLayoutData(fdl); + PropsUi.setLook(combo); + combo.addModifyListener(lsMod); + combo.addFocusListener( + new FocusAdapter() { + @Override + public void focusGained(FocusEvent e) { + populateInputFields(); + } + }); + FormData fd = new FormData(); + fd.left = new FormAttachment(middle, 0); + fd.top = new FormAttachment(previous, margin); + fd.right = new FormAttachment(100, 0); + combo.setLayoutData(fd); + return combo; + } + + private void populateInputFields() { Review Comment: **[bug]** Focusing any of the three field combos calls `removeAll()` on all of them and only restores `wInputField`. The source document ID and content type selections are cleared; OK then persists empty field names. **Suggestion:** Snapshot each combo's text before `removeAll()`, or refresh only the focused combo, then restore all three. ########## plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerDialog.java: ########## @@ -0,0 +1,395 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.chunker; + +import org.apache.hop.core.Const; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType; +import org.apache.hop.pipeline.transforms.chunker.document.ContentType; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.widget.ComboVar; +import org.apache.hop.ui.core.widget.LabelTextVar; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CCombo; +import org.eclipse.swt.events.FocusAdapter; +import org.eclipse.swt.events.FocusEvent; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Shell; + +public class TextChunkerDialog extends BaseTransformDialog { + + private static final Class<?> PKG = TextChunkerMeta.class; + + private final TextChunkerMeta input; + + private ComboVar wInputField; + private ComboVar wSourceDocumentIdField; + private LabelTextVar wOutputChunkField; + private CCombo wChunkingStrategy; + private CCombo wContentType; + private ComboVar wContentTypeField; + private LabelTextVar wChunkSize; + private LabelTextVar wChunkOverlap; + private Button wIncludeMetadata; + private LabelTextVar wChunkIndexField; + private LabelTextVar wChunkStartPosField; + private LabelTextVar wDocumentIdField; + private LabelTextVar wChunkCountField; + + public TextChunkerDialog( + Shell parent, + IVariables variables, + TextChunkerMeta transformMeta, + PipelineMeta pipelineMeta) { + super(parent, variables, transformMeta, pipelineMeta); + input = transformMeta; + } + + @Override + public String open() { Review Comment: **[suggestion]** The dialog is a flat `FormAttachment` stack with the last widget pinned to `wOk`. That layout fights the button bar on resize. `TextChunkerMeta` has no `@GuiPlugin` / `@GuiWidgetElement` annotations. New transform dialogs should use grouped `GuiCompositeWidgets` (`DataSetOutputDialog`). **Suggestion:** Annotate fields with `@GuiWidgetElement` (`BOXES` or `TABS`), then `GuiCompositeWidgets.addScrolledComposite(shell, variables, wTransformName, wOk, PARENT_ID, input)`. Put stream-field combos in `registerExtraGroup(...)` if needed. Use a listener for structure/metadata enablement. ########## plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunker.java: ########## @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.chunker; + +import java.util.List; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.RowDataUtil; +import org.apache.hop.core.util.Utils; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategy; +import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyFactory; +import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType; +import org.apache.hop.pipeline.transforms.chunker.chunking.StructureChunkingStrategy; +import org.apache.hop.pipeline.transforms.chunker.document.ContentType; +import org.apache.hop.pipeline.transforms.chunker.document.ContentTypeResolver; + +/** + * Text Chunker transform - splits text into chunks for AI/ML processing. Supports multiple chunking + * strategies (character-based with word boundary respect, paragraph-based, structure-aware) with + * configurable overlap. + */ +public class TextChunker extends BaseTransform<TextChunkerMeta, TextChunkerData> { + + private static final Class<?> PKG = TextChunkerMeta.class; + + /** The chunking strategy instance. */ + private ChunkingStrategy strategy; + + /** + * Counter used to synthesise a document ID when no source field is configured. Prefixed with the + * transform copy number so parallel copies cannot produce colliding identifiers. + */ + private long documentIdCounter = 0; + + public TextChunker( + TransformMeta transformMeta, + TextChunkerMeta meta, + TextChunkerData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + @Override + public boolean init() { + if (!super.init()) { + return false; + } + strategy = ChunkingStrategyFactory.createStrategy(meta.getChunkingStrategy()); + logBasic( + BaseMessages.getString( + PKG, "TextChunker.Log.Initialized", String.valueOf(meta.getChunkingStrategy()))); + return true; + } + + @Override + public boolean processRow() throws HopException { + Object[] row = getRow(); + + if (row == null) { + setOutputDone(); + return false; + } + + if (first) { + first = false; + data.inputRowMeta = getInputRowMeta(); + data.outputRowMeta = data.inputRowMeta.clone(); + meta.getFields( + data.outputRowMeta, getTransformName(), null, null, this, getMetadataProvider()); + resolveFieldIndices(); + } + + try { + chunkRow(row); + } catch (HopException e) { + if (getTransformMeta().isDoingErrorHandling()) { + putError(data.inputRowMeta, row, 1, e.getMessage(), meta.getInputField(), "TEXTCHUNKER001"); + } else { + throw e; + } + } + + return true; + } + + private void chunkRow(Object[] row) throws HopException { + String text = data.inputRowMeta.getString(row, data.inputFieldIndex); + String documentIdValue = resolveDocumentId(row); + + if (Utils.isEmpty(text)) { + if (isRowLevel()) { + logRowlevel(BaseMessages.getString(PKG, "TextChunker.Log.EmptyText")); + } + putRow(data.outputRowMeta, createOutputRow(row, "", 0, 0, documentIdValue, 0)); + return; + } + + List<Chunk> chunks = chunk(row, text); Review Comment: **[suggestion]** Empty text still emits one row. `CharacterChunkingStrategy` / `StructureChunkingStrategy` return an empty list when `maxSize <= 0`, so this loop outputs nothing and the input row is dropped. Metadata injection can set `chunkSize` to 0 even though `check()` reports an error. Paragraph strategy instead emits the whole text. **Suggestion:** Fail `init`/`processRow` when the resolved chunk size is not positive, or treat a zero-chunk result as a single passthrough row. ########## plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ParagraphChunkingStrategy.java: ########## @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.chunker.chunking; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.hop.pipeline.transforms.chunker.Chunk; + +/** + * Chunking strategy that splits text on paragraph boundaries. + * + * <p>Paragraphs are identified by blank lines. A paragraph that on its own exceeds {@code maxSize} + * is split further with {@link CharacterChunkingStrategy}, because a chunk larger than the limit + * would be rejected downstream by the embedding model rather than simply being large. + * + * <p>Consecutive very short paragraphs are grouped together while they fit within {@code maxSize}. + */ +public class ParagraphChunkingStrategy implements ChunkingStrategy { + + /** + * A blank line. {@code \\R} matches any line ending, so positions stay valid in the original text + * and no CRLF normalisation pass is needed. + */ + private static final Pattern PARAGRAPH_SEPARATOR = Pattern.compile("\\R\\s*\\R"); + + private static final int SHORT_PARAGRAPH_THRESHOLD = 10; Review Comment: **[bug]** Only paragraphs shorter than 10 characters are concatenated. Two 40-character paragraphs with `chunkSize=1000` become two chunks. The user manual says "packing whole paragraphs up to the chunk size", which is the usual embedding-chunker contract. **Suggestion:** Append the next paragraph while `current.length() + 2 + next.length() <= maxSize`. Keep the oversized-paragraph character fallback. Update the golden CSV if packing changes. ########## plugins/transforms/textchunker/src/main/resources/org/apache/hop/pipeline/transforms/chunker/messages/messages_en_US.properties: ########## @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +TextChunkerDialog.Shell.Title=Text chunker +TextChunkerDialog.GetFields.Error.Title=Error getting fields +TextChunkerDialog.GetFields.Error.Message=Unable to read fields from the previous transform + +TextChunker.Name=Text chunker +TextChunker.Description=Split text into chunks for AI/ML processing without breaking words or sentences +TextChunker.Keywords=AI,LLM,text,chunk,split,paragraph,character,word,embedding + +TextChunker.inputField.Label=Input field +TextChunker.inputField.Tooltip=Select the field containing text to chunk + +TextChunker.sourceDocumentIdField.Label=Source document ID field +TextChunker.sourceDocumentIdField.Tooltip=Optional input field to use as the document identifier in metadata. When empty, a row counter is used. + +TextChunker.outputChunkField.Label=Output chunk field +TextChunker.outputChunkField.Tooltip=Name of the field to output chunks to + +TextChunker.chunkingStrategy.Label=Chunking strategy +TextChunker.chunkingStrategy.Tooltip=Choose how to split text: Character, Paragraph, or Structure (heading-aware with breadcrumb prefixes) + +TextChunker.contentType.Label=Content type +TextChunker.contentType.Tooltip=Document format for Structure strategy. Auto detects from source_type field or text heuristics. + +TextChunker.contentTypeField.Label=Content type field +TextChunker.contentTypeField.Tooltip=Optional input field (e.g. source_type) to select parser: doc=AsciiDoc, pipeline/workflow=Hop XML, article/blog=Plain + +TextChunker.chunkSize.Label=Chunk size +TextChunker.chunkSize.Tooltip=Maximum size for each chunk in characters (approximate for paragraphs) + +TextChunker.chunkOverlap.Label=Chunk overlap +TextChunker.chunkOverlap.Tooltip=Number of characters to overlap between chunks (for continuity) + +TextChunker.includeMetadata.Label=Include metadata +TextChunker.includeMetadata.Tooltip=Add metadata fields (chunk index, position, document ID) to output + +TextChunker.chunkIndexField.Label=Chunk index field +TextChunker.chunkIndexField.Tooltip=Name of the field for chunk index (0-based) + +TextChunker.chunkStartPosField.Label=Chunk start position field +TextChunker.chunkStartPosField.Tooltip=Name of the field for chunk start position in original text + +TextChunker.documentIdField.Label=Document ID field +TextChunker.documentIdField.Tooltip=Name of the field for document ID (unique per input row) + +TextChunker.chunkCountField.Label=Total chunks field +TextChunker.chunkCountField.Tooltip=Name of the field for total number of chunks per document + +ChunkingStrategy.CHARACTER=Character +ChunkingStrategy.PARAGRAPH=Paragraph +ChunkingStrategy.STRUCTURE=Structure + +TextChunker.Validation.InputFieldRequired=Input field must be specified +TextChunker.Validation.ChunkSizePositive=Chunk size must be greater than 0 +TextChunker.Validation.OverlapNonNegative=Chunk overlap cannot be negative +TextChunker.Validation.OverlapWarning=Chunk overlap should be less than chunk size for best results +TextChunker.Validation.InputFieldNotFound=Input field ''{0}'' not found in the input stream + +TextChunker.Error.ReadingInputField=Error reading input field +TextChunker.Error.CreatingOutputRow=Error creating output row +TextChunker.Error.UnknownStrategy=Unknown chunking strategy: {0} + +TextChunker.Log.Initialized=Initialized Text Chunker with strategy: {0} +TextChunker.Log.ProcessedDocument=Processed document with {0} chunks +TextChunker.Log.EmptyText=Empty text received, skipping Review Comment: **[nit]** Message says "skipping" but `chunkRow` still `putRow`s an empty chunk. **Suggestion:** Change the message to say an empty chunk row is emitted. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
