This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new c21a5c1997 Text chunker: existing chunk_index/total_chunks fields get
Long values, and an invalid overlap silently becomes 0, fixes #9285 (#9367)
c21a5c1997 is described below
commit c21a5c1997de63eaf109e5f14d2ed513bf146633
Author: Bart Maertens <[email protected]>
AuthorDate: Thu Nov 12 10:42:04 2026 +0100
Text chunker: existing chunk_index/total_chunks fields get Long values, and
an invalid overlap silently becomes 0, fixes #9285 (#9367)
* Text chunker: existing chunk_index/total_chunks fields get Long values,
and an invalid overlap silently becomes 0, fixes #9285
* Text chunker: address review on #9367, a negative overlap stops the
transform, existing fields take the output metadata, #9285
---
.../pages/pipeline/transforms/textchunker.adoc | 4 +
.../0126-text-chunker-existing-fields.hpl | 176 +++++++++++++
.../0126-text-chunker-invalid-overlap.hpl | 130 +++++++++
.../golden-text-chunker-existing-fields.csv | 5 +
.../main-0126-text-chunker-existing-fields.hwf | 79 ++++++
.../main-0126-text-chunker-invalid-overlap.hwf | 292 +++++++++++++++++++++
.../golden-text-chunker-existing-fields.json | 39 +++
.../0126-text-chunker-existing-fields UNIT.json | 38 +++
.../pipeline/transforms/chunker/TextChunker.java | 19 +-
.../transforms/chunker/TextChunkerMeta.java | 124 ++++++++-
.../chunker/messages/messages_en_US.properties | 6 +-
.../transforms/chunker/TextChunkerMetaTest.java | 238 +++++++++++++++++
.../transforms/chunker/TextChunkerTest.java | 95 +++++++
13 files changed, 1223 insertions(+), 22 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc
index f01ed986ae..6053beaf36 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc
@@ -95,4 +95,8 @@ The chunk start position is the character offset in the input
text. For a sectio
All strategies drop chunks that contain only whitespace. The chunk index and
total chunk count cover only the chunks that are emitted. Text that yields no
chunks produces a single row with an empty chunk and a total of 0, the same as
empty text.
+Chunk size must be a whole number greater than 0. Chunk overlap must be a
whole number of 0 or more; leave it empty for no overlap. A value that is not a
number, for example `abc`, or a negative overlap is reported in the pipeline
checks and stops the transform when the pipeline starts.
+
+When the input already has a field with the same name as an output field
(names are matched without regard to case), the transform writes its values
into that field instead of adding a new one. The field keeps its name and
position but takes the transform's type, storage and format. If the types
differ, for example a String `chunk_index` in the input, the pipeline checks
report a warning. Rename the output field to keep the input field as it is.
+
Keep *Include metadata* enabled when the chunks are going into a vector store:
the document ID and chunk index are what let a later search merge neighbouring
chunks back into a passage, and re-index a single document without touching the
rest.
diff --git a/integration-tests/transforms/0126-text-chunker-existing-fields.hpl
b/integration-tests/transforms/0126-text-chunker-existing-fields.hpl
new file mode 100644
index 0000000000..1879864193
--- /dev/null
+++ b/integration-tests/transforms/0126-text-chunker-existing-fields.hpl
@@ -0,0 +1,176 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0126-text-chunker-existing-fields</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>The input already has String chunk_index and total_chunks
fields: they must become Integer fields holding the chunk metadata, and sort
numerically (#9285).</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/07 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/07 12:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop> <from>Document</from> <to>Chunk</to> <enabled>Y</enabled> </hop>
+ <hop> <from>Chunk</from> <to>Sort on chunk_index</to> <enabled>Y</enabled>
</hop>
+ <hop> <from>Sort on chunk_index</from> <to>OUTPUT</to>
<enabled>Y</enabled> </hop>
+ </order>
+ <transform>
+ <name>Document</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>Apache Hop is a data orchestration platform for building
pipelines and workflows.</item>
+ <item>old</item>
+ <item>old</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>content</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>chunk_index</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>total_chunks</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Chunk</name>
+ <type>TextChunker</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <inputField>content</inputField>
+ <outputChunkField>chunk_text</outputChunkField>
+ <chunkingStrategy>CHARACTER</chunkingStrategy>
+ <chunkSize>30</chunkSize>
+ <chunkOverlap>0</chunkOverlap>
+ <includeMetadata>Y</includeMetadata>
+ <chunkIndexField>chunk_index</chunkIndexField>
+ <chunkStartPosField>chunk_start_position</chunkStartPosField>
+ <documentIdField>chunk_doc_id</documentIdField>
+ <chunkCountField>total_chunks</chunkCountField>
+ <attributes/>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sort on chunk_index</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>out</prefix>
+ <sort_size>1000000</sort_size>
+ <free_memory/>
+ <compress>N</compress>
+ <compress_variable/>
+ <unique_rows>N</unique_rows>
+ <fields>
+ <field>
+ <name>chunk_index</name>
+ <ascending>N</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>544</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>OUTPUT</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>768</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0126-text-chunker-invalid-overlap.hpl
b/integration-tests/transforms/0126-text-chunker-invalid-overlap.hpl
new file mode 100644
index 0000000000..e7d66ec762
--- /dev/null
+++ b/integration-tests/transforms/0126-text-chunker-invalid-overlap.hpl
@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0126-text-chunker-invalid-overlap</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Expected to fail: a chunk overlap that is not a number or
negative must stop the transform instead of running with overlap 0
(#9285).</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ <parameter>
+ <name>CHUNK_OVERLAP</name>
+ <default_value>abc</default_value>
+ <description>Invalid chunk overlap to test</description>
+ </parameter>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/07 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/07 12:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop> <from>Document</from> <to>Chunk</to> <enabled>Y</enabled> </hop>
+ <hop> <from>Chunk</from> <to>OUTPUT</to> <enabled>Y</enabled> </hop>
+ </order>
+ <transform>
+ <name>Document</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>Apache Hop is a data orchestration platform for building
pipelines and workflows.</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>content</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Chunk</name>
+ <type>TextChunker</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <inputField>content</inputField>
+ <outputChunkField>chunk_text</outputChunkField>
+ <chunkingStrategy>CHARACTER</chunkingStrategy>
+ <chunkSize>30</chunkSize>
+ <chunkOverlap>${CHUNK_OVERLAP}</chunkOverlap>
+ <includeMetadata>Y</includeMetadata>
+ <chunkIndexField>chunk_index</chunkIndexField>
+ <chunkStartPosField>chunk_start_position</chunkStartPosField>
+ <documentIdField>chunk_doc_id</documentIdField>
+ <chunkCountField>total_chunks</chunkCountField>
+ <attributes/>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>OUTPUT</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>544</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-text-chunker-existing-fields.csv
b/integration-tests/transforms/datasets/golden-text-chunker-existing-fields.csv
new file mode 100644
index 0000000000..8847c1076a
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-text-chunker-existing-fields.csv
@@ -0,0 +1,5 @@
+chunk_text,chunk_index,chunk_start_position,total_chunks
+"workflows.",3,71,4
+"building pipelines and ",2,48,4
+"orchestration platform for ",1,21,4
+"Apache Hop is a data ",0,0,4
diff --git
a/integration-tests/transforms/main-0126-text-chunker-existing-fields.hwf
b/integration-tests/transforms/main-0126-text-chunker-existing-fields.hwf
new file mode 100644
index 0000000000..4c9cb66b96
--- /dev/null
+++ b/integration-tests/transforms/main-0126-text-chunker-existing-fields.hwf
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0126-text-chunker-existing-fields</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Text chunker: input fields named like the chunk metadata fields
take the chunker's Integer type and sort numerically (#9285)</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/07 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/07 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>START</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>96</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0126-text-chunker-existing-fields UNIT</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0126-text-chunker-existing-fields UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>320</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>START</from>
+ <to>0126-text-chunker-existing-fields UNIT</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/main-0126-text-chunker-invalid-overlap.hwf
b/integration-tests/transforms/main-0126-text-chunker-invalid-overlap.hwf
new file mode 100644
index 0000000000..49b632bfa7
--- /dev/null
+++ b/integration-tests/transforms/main-0126-text-chunker-invalid-overlap.hwf
@@ -0,0 +1,292 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0126-text-chunker-invalid-overlap</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Text chunker: a chunk overlap that is not a number or negative
stops the transform instead of running with overlap 0 (#9285)</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/07 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/07 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>START</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>96</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+
<filename>${PROJECT_HOME}/0126-text-chunker-invalid-overlap.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <create_parent_folder>N</create_parent_folder>
+ <set_logfile>Y</set_logfile>
+ <logfile>${java.io.tmpdir}/0126-text-chunker-overlap-abc</logfile>
+ <logext>log</logext>
+ <set_append_logfile>N</set_append_logfile>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <parameters>
+ <parameter>
+ <name>CHUNK_OVERLAP</name>
+ <stream_name/>
+ <value>abc</value>
+ </parameter>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <run_configuration>local</run_configuration>
+ <name>Overlap abc</name>
+ <description>Expected to fail</description>
+ <type>PIPELINE</type>
+ <attributes/>
+ <xloc>320</xloc>
+ <yloc>80</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>abc message?</name>
+ <description/>
+ <type>SHELL</type>
+ <attributes/>
+ <filename/>
+ <work_directory/>
+ <arg_from_previous>N</arg_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <set_append_logfile>N</set_append_logfile>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <insertScript>Y</insertScript>
+ <script>#!/bin/bash
+grep -F "Chunk overlap 'abc' is not a whole number"
"${java.io.tmpdir}/0126-text-chunker-overlap-abc.log"</script>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>320</xloc>
+ <yloc>224</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+
<filename>${PROJECT_HOME}/0126-text-chunker-invalid-overlap.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <create_parent_folder>N</create_parent_folder>
+ <set_logfile>Y</set_logfile>
+ <logfile>${java.io.tmpdir}/0126-text-chunker-overlap-negative</logfile>
+ <logext>log</logext>
+ <set_append_logfile>N</set_append_logfile>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <parameters>
+ <parameter>
+ <name>CHUNK_OVERLAP</name>
+ <stream_name/>
+ <value>-1</value>
+ </parameter>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <run_configuration>local</run_configuration>
+ <name>Overlap -1</name>
+ <description>Expected to fail</description>
+ <type>PIPELINE</type>
+ <attributes/>
+ <xloc>544</xloc>
+ <yloc>80</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>-1 message?</name>
+ <description/>
+ <type>SHELL</type>
+ <attributes/>
+ <filename/>
+ <work_directory/>
+ <arg_from_previous>N</arg_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <set_append_logfile>N</set_append_logfile>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <insertScript>Y</insertScript>
+ <script>#!/bin/bash
+grep -F "Chunk overlap '-1' cannot be negative"
"${java.io.tmpdir}/0126-text-chunker-overlap-negative.log"</script>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>544</xloc>
+ <yloc>224</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Overlap 'abc' did not fail - Abort</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <message>A chunk overlap of 'abc' did not stop the Text chunker</message>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>320</xloc>
+ <yloc>368</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>No message for 'abc' - Abort</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <message>The Text chunker did not log why an overlap of 'abc' is
invalid</message>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>96</xloc>
+ <yloc>224</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Overlap -1 did not fail - Abort</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <message>A chunk overlap of -1 did not stop the Text chunker</message>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>544</xloc>
+ <yloc>368</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>No message for -1 - Abort</name>
+ <description/>
+ <type>ABORT</type>
+ <attributes/>
+ <message>The Text chunker did not log why an overlap of -1 is
invalid</message>
+ <loglevel>Basic</loglevel>
+ <parallel>N</parallel>
+ <xloc>768</xloc>
+ <yloc>224</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Success</name>
+ <description/>
+ <type>SUCCESS</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <xloc>768</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>START</from>
+ <to>Overlap abc</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Overlap abc</from>
+ <to>Overlap 'abc' did not fail - Abort</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Overlap abc</from>
+ <to>abc message?</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>abc message?</from>
+ <to>No message for 'abc' - Abort</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>abc message?</from>
+ <to>Overlap -1</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Overlap -1</from>
+ <to>Overlap -1 did not fail - Abort</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>Overlap -1</from>
+ <to>-1 message?</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>-1 message?</from>
+ <to>No message for -1 - Abort</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>-1 message?</from>
+ <to>Success</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-text-chunker-existing-fields.json
b/integration-tests/transforms/metadata/dataset/golden-text-chunker-existing-fields.json
new file mode 100644
index 0000000000..577597dd90
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-text-chunker-existing-fields.json
@@ -0,0 +1,39 @@
+{
+ "base_filename": "golden-text-chunker-existing-fields.csv",
+ "name": "golden-text-chunker-existing-fields",
+ "description": "Existing String chunk_index and total_chunks fields become
Integer chunk metadata, sorted on chunk_index descending",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "chunk_text",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "chunk_index",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "chunk_start_position",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "total_chunks",
+ "field_format": "0"
+ }
+ ]
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0126-text-chunker-existing-fields
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0126-text-chunker-existing-fields
UNIT.json
new file mode 100644
index 0000000000..846fd52977
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0126-text-chunker-existing-fields
UNIT.json
@@ -0,0 +1,38 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": false,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "chunk_text",
+ "data_set_field": "chunk_text"
+ },
+ {
+ "transform_field": "chunk_index",
+ "data_set_field": "chunk_index"
+ },
+ {
+ "transform_field": "chunk_start_position",
+ "data_set_field": "chunk_start_position"
+ },
+ {
+ "transform_field": "total_chunks",
+ "data_set_field": "total_chunks"
+ }
+ ],
+ "field_order": [],
+ "transform_name": "OUTPUT",
+ "data_set_name": "golden-text-chunker-existing-fields"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0126-text-chunker-existing-fields UNIT",
+ "description": "Existing String chunk_index and total_chunks fields become
Integer chunk metadata and sort numerically",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0126-text-chunker-existing-fields.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunker.java
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunker.java
index 612060a191..a6e3af7fde 100644
---
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunker.java
+++
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunker.java
@@ -17,7 +17,6 @@
package org.apache.hop.pipeline.transforms.chunker;
import java.util.List;
-import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.RowDataUtil;
import org.apache.hop.core.util.Utils;
@@ -67,17 +66,17 @@ public class TextChunker extends
BaseTransform<TextChunkerMeta, TextChunkerData>
if (!super.init()) {
return false;
}
- data.chunkSize = Const.toInt(resolve(meta.getChunkSize()), -1);
- data.chunkOverlap = Const.toInt(resolve(meta.getChunkOverlap()), -1);
- if (data.chunkSize <= 0) {
- logError(
- BaseMessages.getString(
- PKG, "TextChunker.Validation.ChunkSizePositive",
meta.getChunkSize()));
+ TextChunkerMeta.ChunkSettings settings =
+ TextChunkerMeta.validateChunkSettings(
+ resolve(meta.getChunkSize()), resolve(meta.getChunkOverlap()));
+ if (!settings.isValid()) {
+ for (String message : settings.errors()) {
+ logError(message);
+ }
return false;
}
- if (data.chunkOverlap < 0) {
- data.chunkOverlap = 0;
- }
+ data.chunkSize = settings.size();
+ data.chunkOverlap = settings.overlap();
strategy =
ChunkingStrategyFactory.createStrategy(meta.getChunkingStrategy());
if (strategy instanceof ParagraphChunkingStrategy paragraphStrategy) {
paragraphStrategy.setSeparator(
diff --git
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMeta.java
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMeta.java
index 4056aaf175..00d3121591 100644
---
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMeta.java
+++
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMeta.java
@@ -17,11 +17,11 @@
package org.apache.hop.pipeline.transforms.chunker;
import java.util.ArrayList;
+import java.util.LinkedHashSet;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.apache.hop.core.CheckResult;
-import org.apache.hop.core.Const;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopPluginException;
@@ -329,14 +329,83 @@ public class TextChunkerMeta extends
BaseTransformMeta<TextChunker, TextChunkerD
}
}
+ /**
+ * Adds an output field. When the input already has a field with that name
(Hop matches field
+ * names without regard to case) the transform writes its values into that
field, so the field
+ * takes the transform's metadata (type, storage, format, length) and keeps
its existing name.
+ */
private static void addField(IRowMeta row, IValueMeta field, String origin)
throws HopPluginException {
String fieldName = field.getName();
- if (fieldName == null || fieldName.isEmpty() ||
row.indexOfValue(fieldName) >= 0) {
+ if (fieldName == null || fieldName.isEmpty()) {
return;
}
field.setOrigin(origin);
- row.addValueMeta(field);
+ int index = row.indexOfValue(fieldName);
+ if (index < 0) {
+ row.addValueMeta(field);
+ } else {
+ field.setName(row.getValueMeta(index).getName());
+ row.setValueMeta(index, field);
+ }
+ }
+
+ /**
+ * Parses a resolved chunk size or overlap.
+ *
+ * @param value the resolved value
+ * @param emptyValue the value to use when nothing is configured
+ * @return the number, or {@code null} when the value is not a whole number
+ */
+ static Integer parseNumber(String value, int emptyValue) {
+ if (value == null || value.isBlank()) {
+ return emptyValue;
+ }
+ try {
+ return Integer.parseInt(value.trim());
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+ /**
+ * The resolved chunk size and overlap, and the errors that stop the
transform. Both {@link
+ * #check} and {@link TextChunker#init()} use this, so the pipeline checks
and the run apply the
+ * same rules.
+ *
+ * @param size the chunk size, or -1 when it is not a whole number
+ * @param overlap the chunk overlap, or -1 when it is not a whole number
+ * @param errors the error messages, empty when both values are valid
+ */
+ record ChunkSettings(int size, int overlap, List<String> errors) {
+ boolean isValid() {
+ return errors.isEmpty();
+ }
+ }
+
+ /**
+ * Validates a resolved chunk size and overlap. The size must be a whole
number greater than 0,
+ * the overlap a whole number of 0 or more. An empty overlap means no
overlap.
+ */
+ static ChunkSettings validateChunkSettings(String sizeText, String
overlapText) {
+ List<String> errors = new ArrayList<>();
+ Integer size = parseNumber(sizeText, 0);
+ Integer overlap = parseNumber(overlapText, 0);
+ if (size == null) {
+ errors.add(
+ BaseMessages.getString(PKG,
"TextChunker.Validation.ChunkSizeNotANumber", sizeText));
+ } else if (size <= 0) {
+ errors.add(BaseMessages.getString(PKG,
"TextChunker.Validation.ChunkSizePositive"));
+ }
+ if (overlap == null) {
+ errors.add(
+ BaseMessages.getString(PKG,
"TextChunker.Validation.OverlapNotANumber", overlapText));
+ } else if (overlap < 0) {
+ errors.add(
+ BaseMessages.getString(PKG,
"TextChunker.Validation.OverlapNonNegative", overlapText));
+ }
+ return new ChunkSettings(
+ size == null ? -1 : size, overlap == null ? -1 : overlap,
List.copyOf(errors));
}
/**
@@ -411,16 +480,16 @@ public class TextChunkerMeta extends
BaseTransformMeta<TextChunker, TextChunkerD
error(remarks, transformMeta,
"TextChunker.Validation.OutputChunkFieldRequired");
}
- int resolvedSize = Const.toInt(variables.resolve(chunkSize), -1);
- int resolvedOverlap = Const.toInt(variables.resolve(chunkOverlap), -1);
+ checkOutputFieldTypes(remarks, transformMeta, prev);
- if (resolvedSize <= 0) {
- error(remarks, transformMeta,
"TextChunker.Validation.ChunkSizePositive");
+ ChunkSettings settings =
+ validateChunkSettings(variables.resolve(chunkSize),
variables.resolve(chunkOverlap));
+ for (String message : settings.errors()) {
+ remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, message,
transformMeta));
}
-
- if (resolvedOverlap < 0) {
- error(remarks, transformMeta,
"TextChunker.Validation.OverlapNonNegative");
- } else if (resolvedSize > 0 && resolvedOverlap >= resolvedSize) {
+ int resolvedSize = settings.size();
+ int resolvedOverlap = settings.overlap();
+ if (resolvedSize > 0 && resolvedOverlap >= resolvedSize) {
warning(remarks, transformMeta, "TextChunker.Validation.OverlapWarning");
}
@@ -439,6 +508,39 @@ public class TextChunkerMeta extends
BaseTransformMeta<TextChunker, TextChunkerD
}
}
+ /** Warns when an output field already exists in the input with a different
type. */
+ private void checkOutputFieldTypes(
+ List<ICheckResult> remarks, TransformMeta transformMeta, IRowMeta prev) {
+ if (prev == null) {
+ return;
+ }
+ IRowMeta output = prev.clone();
+ try {
+ getFields(output, transformMeta.getName(), null, null, null, null);
+ } catch (HopTransformException e) {
+ error(remarks, transformMeta,
"TextChunker.Validation.OutputFieldsError", e.getMessage());
+ return;
+ }
+ for (String fieldName : new LinkedHashSet<>(getOutputFieldNames())) {
+ if (Utils.isEmpty(fieldName)) {
+ continue;
+ }
+ IValueMeta inputField = prev.searchValueMeta(fieldName);
+ IValueMeta outputField = output.searchValueMeta(fieldName);
+ if (inputField != null
+ && outputField != null
+ && inputField.getType() != outputField.getType()) {
+ warning(
+ remarks,
+ transformMeta,
+ "TextChunker.Validation.OutputFieldTypeChanged",
+ fieldName,
+ inputField.getTypeDesc(),
+ outputField.getTypeDesc());
+ }
+ }
+ }
+
private static void error(
List<ICheckResult> remarks, TransformMeta transformMeta, String key,
String... parameters) {
remarks.add(
diff --git
a/plugins/transforms/textchunker/src/main/resources/org/apache/hop/pipeline/transforms/chunker/messages/messages_en_US.properties
b/plugins/transforms/textchunker/src/main/resources/org/apache/hop/pipeline/transforms/chunker/messages/messages_en_US.properties
index 37df05af3d..76425a6763 100644
---
a/plugins/transforms/textchunker/src/main/resources/org/apache/hop/pipeline/transforms/chunker/messages/messages_en_US.properties
+++
b/plugins/transforms/textchunker/src/main/resources/org/apache/hop/pipeline/transforms/chunker/messages/messages_en_US.properties
@@ -70,7 +70,11 @@ ChunkingStrategy.STRUCTURE=Structure
TextChunker.Validation.InputFieldRequired=Input field must be specified
TextChunker.Validation.ChunkSizePositive=Chunk size must be greater than 0
-TextChunker.Validation.OverlapNonNegative=Chunk overlap cannot be negative
+TextChunker.Validation.OverlapNonNegative=Chunk overlap ''{0}'' cannot be
negative
+TextChunker.Validation.ChunkSizeNotANumber=Chunk size ''{0}'' is not a whole
number
+TextChunker.Validation.OverlapNotANumber=Chunk overlap ''{0}'' is not a whole
number
+TextChunker.Validation.OutputFieldTypeChanged=Output field ''{0}'' already
exists in the input as {1}; the transform writes {2} values into it, so the
field becomes {2}
+TextChunker.Validation.OutputFieldsError=Cannot determine the output fields:
{0}
TextChunker.Validation.OverlapWarning=Chunk overlap should be less than chunk
size for best results
TextChunker.Validation.InputFieldNotFound=Input field ''{0}'' not found in the
input stream
diff --git
a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMetaTest.java
b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMetaTest.java
index d1285beaef..76ddb35eb1 100644
---
a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMetaTest.java
+++
b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMetaTest.java
@@ -17,18 +17,23 @@
package org.apache.hop.pipeline.transforms.chunker;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.ArrayList;
import java.util.List;
import org.apache.hop.core.HopClientEnvironment;
import org.apache.hop.core.ICheckResult;
+import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.core.xml.XmlHandler;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil;
import org.apache.hop.pipeline.transform.TransformMeta;
import
org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType;
@@ -158,6 +163,239 @@ class TextChunkerMetaTest {
assertEquals(IValueMeta.TYPE_STRING,
row.searchValueMeta("chunk_doc_id").getType());
}
+ /**
+ * The transform writes Long values into an input field that has the same
name as one of its
+ * output fields, so that field must take the transform's type (#9285).
+ */
+ @Test
+ void anExistingFieldTakesTheOutputType() throws Exception {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+
+ IRowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaString("text"));
+ row.addValueMeta(new ValueMetaString("chunk_index"));
+ row.addValueMeta(new ValueMetaString("total_chunks"));
+ meta.getFields(row, "chunker", null, null, new Variables(), null);
+
+ assertEquals(6, row.size(), "existing fields are reused, not duplicated");
+ assertEquals(1, row.indexOfValue("chunk_index"), "the field keeps its
position");
+ assertEquals(IValueMeta.TYPE_INTEGER, row.getValueMeta(1).getType());
+ assertEquals(IValueMeta.TYPE_INTEGER, row.getValueMeta(2).getType());
+ }
+
+ /**
+ * The transform writes plain String values, so an existing field of the
same type must not keep
+ * its own storage type, mask or length. With lazy conversion the input
field is a binary string;
+ * keeping that metadata would make later transforms read a String as byte[].
+ */
+ @Test
+ void anExistingFieldOfTheSameTypeTakesTheOutputMetadata() throws Exception {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("body");
+
+ IRowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaString("body"));
+ ValueMetaString lazy = new ValueMetaString("chunk_text", 5000, -1);
+ lazy.setStorageType(IValueMeta.STORAGE_TYPE_BINARY_STRING);
+ lazy.setStorageMetadata(new ValueMetaString("chunk_text"));
+ lazy.setConversionMask("#");
+ row.addValueMeta(lazy);
+ meta.getFields(row, "chunker", null, null, new Variables(), null);
+
+ IValueMeta chunkText = row.getValueMeta(1);
+ assertEquals("chunk_text", chunkText.getName());
+ assertEquals(IValueMeta.STORAGE_TYPE_NORMAL, chunkText.getStorageType());
+ assertEquals(-1, chunkText.getLength());
+ assertNull(chunkText.getConversionMask());
+ }
+
+ /**
+ * Hop matches field names without regard to case. An input field named
Chunk_Index is reused, and
+ * keeps its name so later transforms that look it up by that name still
find it.
+ */
+ @Test
+ void anExistingFieldKeepsItsName() throws Exception {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+
+ IRowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaString("text"));
+ row.addValueMeta(new ValueMetaString("Chunk_Index"));
+ meta.getFields(row, "chunker", null, null, new Variables(), null);
+
+ assertEquals(6, row.size(), "the existing field is reused, not
duplicated");
+ assertEquals("Chunk_Index", row.getValueMeta(1).getName());
+ assertEquals(IValueMeta.TYPE_INTEGER, row.getValueMeta(1).getType());
+ }
+
+ /** Two output fields with the same name give one warning, not two. */
+ @Test
+ void checkWarnsOnceForADuplicateOutputFieldName() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("body");
+ meta.setChunkIndexField("n");
+ meta.setChunkCountField("n");
+
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaString("body"));
+ prev.addValueMeta(new ValueMetaString("n"));
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(remarks, null, new TransformMeta(), prev, null, null, null, new
Variables(), null);
+
+ assertEquals(
+ 1,
+ remarks.stream()
+ .filter(
+ r -> r.getType() == ICheckResult.TYPE_RESULT_WARNING &&
r.getText().contains("'n'"))
+ .count());
+ }
+
+ /** When the output fields cannot be determined, the checks say so instead
of staying silent. */
+ @Test
+ void checkReportsAnErrorWhenTheOutputFieldsCannotBeDetermined() {
+ TextChunkerMeta meta =
+ new TextChunkerMeta() {
+ @Override
+ public void getFields(
+ IRowMeta row,
+ String origin,
+ IRowMeta[] info,
+ TransformMeta nextTransform,
+ IVariables variables,
+ IHopMetadataProvider metadataProvider)
+ throws HopTransformException {
+ throw new HopTransformException("broken field");
+ }
+ };
+ meta.setDefault();
+ meta.setInputField("body");
+
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaString("body"));
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(remarks, null, new TransformMeta(), prev, null, null, null, new
Variables(), null);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_ERROR
+ && r.getText().contains("broken field")),
+ "a getFields failure must be reported");
+ }
+
+ /** The checks and the transform share one set of rules for the chunk size
and overlap. */
+ @Test
+ void validatesChunkSettings() {
+ assertTrue(TextChunkerMeta.validateChunkSettings("100", "").isValid());
+ assertEquals(0, TextChunkerMeta.validateChunkSettings("100",
"").overlap());
+ assertTrue(TextChunkerMeta.validateChunkSettings("100", "0").isValid());
+ assertFalse(TextChunkerMeta.validateChunkSettings("100", "-1").isValid());
+ assertFalse(TextChunkerMeta.validateChunkSettings("100", "abc").isValid());
+ assertFalse(TextChunkerMeta.validateChunkSettings("0", "0").isValid());
+ assertFalse(TextChunkerMeta.validateChunkSettings("", "0").isValid());
+ assertEquals(2, TextChunkerMeta.validateChunkSettings("abc",
"-1").errors().size());
+ }
+
+ @Test
+ void checkWarnsWhenAnExistingFieldChangesType() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("body");
+
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaString("body"));
+ prev.addValueMeta(new ValueMetaString("chunk_index"));
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(remarks, null, new TransformMeta(), prev, null, null, null, new
Variables(), null);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_WARNING
+ && r.getText().contains("chunk_index")),
+ "a String chunk_index in the input must produce a warning");
+ }
+
+ @Test
+ void checkReportsAnOverlapThatIsNotANumber() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("body");
+ meta.setChunkOverlap("abc");
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(remarks, null, new TransformMeta(), null, null, null, null, new
Variables(), null);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_ERROR
+ && r.getText().contains("'abc' is not a whole
number")),
+ "an overlap of 'abc' must be reported as not a number");
+ assertFalse(
+ remarks.stream().anyMatch(r -> r.getText().contains("negative")),
+ "an overlap of 'abc' is not negative");
+ }
+
+ @Test
+ void checkReportsANegativeOverlap() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("body");
+ meta.setChunkOverlap("-5");
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(remarks, null, new TransformMeta(), null, null, null, null, new
Variables(), null);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_ERROR
+ && r.getText().contains("'-5' cannot be negative")),
+ "an overlap of -5 must be reported as negative");
+ }
+
+ @Test
+ void checkReportsAChunkSizeThatIsNotANumber() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("body");
+ meta.setChunkSize("abc");
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(remarks, null, new TransformMeta(), null, null, null, null, new
Variables(), null);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_ERROR
+ && r.getText().contains("'abc' is not a whole
number")),
+ "a chunk size of 'abc' must be reported as not a number");
+ }
+
+ @Test
+ void parsesSizeAndOverlap() {
+ assertEquals(100, TextChunkerMeta.parseNumber(" 100 ", 0));
+ assertEquals(0, TextChunkerMeta.parseNumber("", 0));
+ assertEquals(0, TextChunkerMeta.parseNumber(null, 0));
+ assertEquals(-5, TextChunkerMeta.parseNumber("-5", 0));
+ assertNull(TextChunkerMeta.parseNumber("abc", 0));
+ assertNull(TextChunkerMeta.parseNumber("1.5", 0));
+ }
+
@Test
void checkReportsMissingInputField() {
TextChunkerMeta meta = new TextChunkerMeta();
diff --git
a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerTest.java
b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerTest.java
index 3fe51678be..198dad423f 100644
---
a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerTest.java
+++
b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerTest.java
@@ -30,6 +30,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hop.core.HopClientEnvironment;
import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.row.value.ValueMetaString;
@@ -244,6 +245,100 @@ class TextChunkerTest {
assertFalse(transform.init(), "a non-positive chunk size must fail init,
not drop rows");
}
+ @Test
+ void initFailsWhenTheOverlapIsNotANumber() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+ meta.setChunkOverlap("abc");
+
+ TextChunker transform =
+ new TextChunker(
+ helper.transformMeta,
+ meta,
+ new TextChunkerData(),
+ 0,
+ helper.pipelineMeta,
+ helper.pipeline);
+
+ assertFalse(transform.init(), "an overlap of 'abc' must fail init, not
become 0");
+ }
+
+ @Test
+ void initFailsWhenTheOverlapIsNegative() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+ meta.setChunkOverlap("-5");
+
+ TextChunker transform =
+ new TextChunker(
+ helper.transformMeta,
+ meta,
+ new TextChunkerData(),
+ 0,
+ helper.pipelineMeta,
+ helper.pipeline);
+
+ assertFalse(transform.init(), "an overlap of -5 must fail init, not become
0");
+ }
+
+ @Test
+ void initFailsWhenTheChunkSizeIsNotANumber() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+ meta.setChunkSize("abc");
+
+ TextChunker transform =
+ new TextChunker(
+ helper.transformMeta,
+ meta,
+ new TextChunkerData(),
+ 0,
+ helper.pipelineMeta,
+ helper.pipeline);
+
+ assertFalse(transform.init(), "a chunk size of 'abc' must fail init");
+ }
+
+ /**
+ * Input fields named like the chunk metadata fields receive Long values, so
the output row
+ * metadata must describe them as Integer (#9285).
+ */
+ @Test
+ void existingMetadataFieldsMatchTheValuesWritten() throws Exception {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+ meta.setChunkSize("10");
+ meta.setChunkOverlap("0");
+
+ IRowMeta inputRowMeta = rowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("chunk_index"));
+ inputRowMeta.addValueMeta(new ValueMetaString("total_chunks"));
+
+ run(
+ meta,
+ inputRowMeta,
+ List.<Object[]>of(new Object[] {"aaaaaaaa bbbbbbbb cccccccc", "old",
"old"}));
+
+ assertTrue(output.size() > 1);
+ for (Object[] row : output) {
+ for (int i = 0; i < outputRowMeta.size(); i++) {
+ IValueMeta valueMeta = outputRowMeta.getValueMeta(i);
+ if (row[i] != null) {
+ assertEquals(
+ valueMeta.getNativeDataTypeClass(),
+ row[i].getClass(),
+ "value of " + valueMeta.getName() + " must match its metadata");
+ }
+ }
+ }
+ assertEquals(IValueMeta.TYPE_INTEGER,
outputRowMeta.searchValueMeta("chunk_index").getType());
+ assertEquals(IValueMeta.TYPE_INTEGER,
outputRowMeta.searchValueMeta("total_chunks").getType());
+ }
+
/** A multi-byte separator configured on the meta must drive the Paragraph
strategy end to end. */
@Test
void splitsOnTheConfiguredParagraphSeparator() throws Exception {