This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 3674b5e70a Improve copy rows to result, fixes #9328 (#9331)
3674b5e70a is described below

commit 3674b5e70a50083fffcb1363a6f4af2406b285e7
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 20:31:57 2026 +0100

    Improve copy rows to result, fixes #9328 (#9331)
---
 .../0004-rows-to-result-validation.hpl             | 282 +++++++++++++++++++++
 .../single_threaded/0004-rows-to-result.hpl        | 122 +++++++++
 .../single_threaded/main-0004-rows-to-result.hwf   | 138 ++++++++++
 .../transforms/rowstoresult/RowsToResult.java      |  46 +++-
 .../transforms/rowstoresult/RowsToResultData.java  |   9 +
 .../transforms/rowstoresult/RowsToResultTest.java  | 235 +++++++++++++++++
 6 files changed, 823 insertions(+), 9 deletions(-)

diff --git 
a/integration-tests/single_threaded/0004-rows-to-result-validation.hpl 
b/integration-tests/single_threaded/0004-rows-to-result-validation.hpl
new file mode 100644
index 0000000000..341f7b4e4f
--- /dev/null
+++ b/integration-tests/single_threaded/0004-rows-to-result-validation.hpl
@@ -0,0 +1,282 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+  <info>
+    <name>0004-rows-to-result-validation</name>
+    <name_sync_with_filename>Y</name_sync_with_filename>
+    <description>Verify the result rows of 0004-rows-to-result, which ran on 
the single threaded engine</description>
+    <extended_description/>
+    <pipeline_version/>
+    <pipeline_type>Normal</pipeline_type>
+    <parameters>
+    </parameters>
+    <capture_transform_performance>N</capture_transform_performance>
+    
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+    
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+    <created_user>-</created_user>
+    <created_date>2026/11/06 12:00:00.000</created_date>
+    <modified_user>-</modified_user>
+    <modified_date>2026/11/06 12:00:00.000</modified_date>
+  </info>
+  <notepads>
+  </notepads>
+  <order>
+    <hop>
+      <from>Expected</from>
+      <to>Merge rows (diff)</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>Output of the test</from>
+      <to>Sort output</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>Sort output</from>
+      <to>Merge rows (diff)</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>Merge rows (diff)</from>
+      <to>Not identical</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>Not identical</from>
+      <to>Abort</to>
+      <enabled>Y</enabled>
+    </hop>
+  </order>
+  <transform>
+    <name>Expected</name>
+    <type>DataGrid</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <fields>
+      <field>
+        <set_empty_string>N</set_empty_string>
+        <length>-1</length>
+        <name>id</name>
+        <precision>-1</precision>
+        <type>Integer</type>
+      </field>
+      <field>
+        <set_empty_string>N</set_empty_string>
+        <length>-1</length>
+        <name>name</name>
+        <precision>-1</precision>
+        <type>String</type>
+      </field>
+    </fields>
+    <data>
+      <line>
+        <item>1</item>
+        <item>alpha</item>
+      </line>
+      <line>
+        <item>2</item>
+        <item>bravo</item>
+      </line>
+      <line>
+        <item>3</item>
+        <item>charlie</item>
+      </line>
+      <line>
+        <item>4</item>
+        <item>delta</item>
+      </line>
+      <line>
+        <item>5</item>
+        <item>echo</item>
+      </line>
+    </data>
+    <attributes/>
+    <GUI>
+      <xloc>96</xloc>
+      <yloc>64</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>Output of the test</name>
+    <type>RowsFromResult</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <fields>
+      <field>
+        <name>id</name>
+        <type>Integer</type>
+        <length>-1</length>
+        <precision>-1</precision>
+      </field>
+      <field>
+        <name>name</name>
+        <type>String</type>
+        <length>-1</length>
+        <precision>-1</precision>
+      </field>
+    </fields>
+    <attributes/>
+    <GUI>
+      <xloc>96</xloc>
+      <yloc>224</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>Sort output</name>
+    <type>SortRows</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <directory>${java.io.tmpdir}</directory>
+    <sort_prefix>out</sort_prefix>
+    <sort_size>1000000</sort_size>
+    <free_memory/>
+    <unique_rows>N</unique_rows>
+    <compress>N</compress>
+    <fields>
+      <field>
+        <name>id</name>
+        <ascending>Y</ascending>
+        <case_sensitive>N</case_sensitive>
+        <collator_enabled>N</collator_enabled>
+        <collator_strength>0</collator_strength>
+        <presorted>N</presorted>
+      </field>
+      <field>
+        <name>name</name>
+        <ascending>Y</ascending>
+        <case_sensitive>N</case_sensitive>
+        <collator_enabled>N</collator_enabled>
+        <collator_strength>0</collator_strength>
+        <presorted>N</presorted>
+      </field>
+    </fields>
+    <attributes/>
+    <GUI>
+      <xloc>256</xloc>
+      <yloc>224</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>Merge rows (diff)</name>
+    <type>MergeRows</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <keys>
+      <key>id</key>
+      <key>name</key>
+    </keys>
+    <values>
+      <value>id</value>
+      <value>name</value>
+    </values>
+    <flag_field>flagfield</flag_field>
+    <reference>Expected</reference>
+    <compare>Sort output</compare>
+    <attributes/>
+    <GUI>
+      <xloc>416</xloc>
+      <yloc>144</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>Not identical</name>
+    <type>FilterRows</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <send_true_to/>
+    <send_false_to/>
+    <compare>
+      <condition>
+        <negated>N</negated>
+        <leftvalue>flagfield</leftvalue>
+        <function>&lt;&gt;</function>
+        <rightvalue/>
+        <value>
+          <name>constant</name>
+          <type>String</type>
+          <text>identical</text>
+          <length>-1</length>
+          <precision>-1</precision>
+          <isnull>N</isnull>
+          <mask/>
+        </value>
+      </condition>
+    </compare>
+    <attributes/>
+    <GUI>
+      <xloc>576</xloc>
+      <yloc>144</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>Abort</name>
+    <type>Abort</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <abort_option>ABORT_WITH_ERROR</abort_option>
+    <always_log_rows>Y</always_log_rows>
+    <message>0004-rows-to-result: the result rows of Copy rows to result on 
the single threaded engine are not the expected ones</message>
+    <row_threshold>0</row_threshold>
+    <attributes/>
+    <GUI>
+      <xloc>736</xloc>
+      <yloc>144</yloc>
+    </GUI>
+  </transform>
+  <transform_error_handling>
+  </transform_error_handling>
+  <attributes/>
+</pipeline>
diff --git a/integration-tests/single_threaded/0004-rows-to-result.hpl 
b/integration-tests/single_threaded/0004-rows-to-result.hpl
new file mode 100644
index 0000000000..7007ccba15
--- /dev/null
+++ b/integration-tests/single_threaded/0004-rows-to-result.hpl
@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+  <info>
+    <name>0004-rows-to-result</name>
+    <name_sync_with_filename>Y</name_sync_with_filename>
+    <description>Copy rows to result on the single threaded engine: the rows 
must reach the result (#9328)</description>
+    <extended_description/>
+    <pipeline_version/>
+    <pipeline_type>Normal</pipeline_type>
+    <parameters>
+    </parameters>
+    <capture_transform_performance>N</capture_transform_performance>
+    
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+    
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+    <created_user>-</created_user>
+    <created_date>2026/11/06 12:00:00.000</created_date>
+    <modified_user>-</modified_user>
+    <modified_date>2026/11/06 12:00:00.000</modified_date>
+  </info>
+  <notepads>
+  </notepads>
+  <order>
+    <hop>
+      <from>rows</from>
+      <to>Copy rows to result</to>
+      <enabled>Y</enabled>
+    </hop>
+  </order>
+  <transform>
+    <name>rows</name>
+    <type>DataGrid</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <fields>
+      <field>
+        <set_empty_string>N</set_empty_string>
+        <length>-1</length>
+        <name>id</name>
+        <precision>-1</precision>
+        <type>Integer</type>
+      </field>
+      <field>
+        <set_empty_string>N</set_empty_string>
+        <length>-1</length>
+        <name>name</name>
+        <precision>-1</precision>
+        <type>String</type>
+      </field>
+    </fields>
+    <data>
+      <line>
+        <item>1</item>
+        <item>alpha</item>
+      </line>
+      <line>
+        <item>2</item>
+        <item>bravo</item>
+      </line>
+      <line>
+        <item>3</item>
+        <item>charlie</item>
+      </line>
+      <line>
+        <item>4</item>
+        <item>delta</item>
+      </line>
+      <line>
+        <item>5</item>
+        <item>echo</item>
+      </line>
+    </data>
+    <attributes/>
+    <GUI>
+      <xloc>96</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>Copy rows to result</name>
+    <type>RowsToResult</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <attributes/>
+    <GUI>
+      <xloc>288</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform_error_handling>
+  </transform_error_handling>
+  <attributes/>
+</pipeline>
diff --git a/integration-tests/single_threaded/main-0004-rows-to-result.hwf 
b/integration-tests/single_threaded/main-0004-rows-to-result.hwf
new file mode 100644
index 0000000000..5c185b2bef
--- /dev/null
+++ b/integration-tests/single_threaded/main-0004-rows-to-result.hwf
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+  <name>main-0004-rows-to-result</name>
+  <name_sync_with_filename>Y</name_sync_with_filename>
+  <description/>
+  <extended_description/>
+  <workflow_version/>
+  <created_user>-</created_user>
+  <created_date>2026/11/06 12:00:00.000</created_date>
+  <modified_user>-</modified_user>
+  <modified_date>2026/11/06 12:00:00.000</modified_date>
+  <parameters>
+    </parameters>
+  <actions>
+    <action>
+      <name>Start</name>
+      <description/>
+      <type>SPECIAL</type>
+      <attributes/>
+      <start>Y</start>
+      <dummy>N</dummy>
+      <repeat>N</repeat>
+      <schedulerType>0</schedulerType>
+      <intervalSeconds>0</intervalSeconds>
+      <intervalMinutes>60</intervalMinutes>
+      <hour>12</hour>
+      <minutes>0</minutes>
+      <weekDay>1</weekDay>
+      <DayOfMonth>1</DayOfMonth>
+      <parallel>N</parallel>
+      <nr>0</nr>
+      <xloc>96</xloc>
+      <yloc>96</yloc>
+      <attributes_hac/>
+    </action>
+    <action>
+      <name>0004-rows-to-result.hpl</name>
+      <description/>
+      <type>PIPELINE</type>
+      <attributes/>
+      <filename>${PROJECT_HOME}/0004-rows-to-result.hpl</filename>
+      <params_from_previous>N</params_from_previous>
+      <exec_per_row>N</exec_per_row>
+      <clear_rows>N</clear_rows>
+      <clear_files>N</clear_files>
+      <set_logfile>N</set_logfile>
+      <logfile/>
+      <logext/>
+      <add_date>N</add_date>
+      <add_time>N</add_time>
+      <loglevel>Basic</loglevel>
+      <set_append_logfile>N</set_append_logfile>
+      <wait_until_finished>Y</wait_until_finished>
+      <follow_abort_remote>N</follow_abort_remote>
+      <create_parent_folder>N</create_parent_folder>
+      <run_configuration>single-threaded</run_configuration>
+      <parameters>
+        <pass_all_parameters>Y</pass_all_parameters>
+      </parameters>
+      <parallel>N</parallel>
+      <nr>0</nr>
+      <xloc>320</xloc>
+      <yloc>96</yloc>
+      <attributes_hac/>
+    </action>
+    <action>
+      <name>0004-rows-to-result-validation.hpl</name>
+      <description/>
+      <type>PIPELINE</type>
+      <attributes/>
+      <filename>${PROJECT_HOME}/0004-rows-to-result-validation.hpl</filename>
+      <params_from_previous>N</params_from_previous>
+      <exec_per_row>N</exec_per_row>
+      <clear_rows>N</clear_rows>
+      <clear_files>N</clear_files>
+      <set_logfile>N</set_logfile>
+      <logfile/>
+      <logext/>
+      <add_date>N</add_date>
+      <add_time>N</add_time>
+      <loglevel>Basic</loglevel>
+      <set_append_logfile>N</set_append_logfile>
+      <wait_until_finished>Y</wait_until_finished>
+      <follow_abort_remote>N</follow_abort_remote>
+      <create_parent_folder>N</create_parent_folder>
+      <run_configuration>local</run_configuration>
+      <parameters>
+        <pass_all_parameters>Y</pass_all_parameters>
+      </parameters>
+      <parallel>N</parallel>
+      <nr>0</nr>
+      <xloc>608</xloc>
+      <yloc>96</yloc>
+      <attributes_hac/>
+    </action>
+  </actions>
+  <hops>
+    <hop>
+      <from>Start</from>
+      <to>0004-rows-to-result.hpl</to>
+      <from_nr>0</from_nr>
+      <to_nr>0</to_nr>
+      <enabled>Y</enabled>
+      <evaluation>Y</evaluation>
+      <unconditional>Y</unconditional>
+    </hop>
+    <hop>
+      <from>0004-rows-to-result.hpl</from>
+      <to>0004-rows-to-result-validation.hpl</to>
+      <from_nr>0</from_nr>
+      <to_nr>0</to_nr>
+      <enabled>Y</enabled>
+      <evaluation>Y</evaluation>
+      <unconditional>N</unconditional>
+    </hop>
+  </hops>
+  <notepads>
+  </notepads>
+  <attributes/>
+</workflow>
diff --git 
a/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResult.java
 
b/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResult.java
index 2aa59ac694..cec044221c 100644
--- 
a/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResult.java
+++ 
b/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResult.java
@@ -17,6 +17,7 @@
 
 package org.apache.hop.pipeline.transforms.rowstoresult;
 
+import java.util.List;
 import org.apache.hop.core.RowMetaAndData;
 import org.apache.hop.core.exception.HopException;
 import org.apache.hop.i18n.BaseMessages;
@@ -45,19 +46,22 @@ public class RowsToResult extends 
BaseTransform<RowsToResultMeta, RowsToResultDa
 
     Object[] r = getRow(); // get row, set busy!
     if (r == null) { // no more input to be expected...
-
-      ((Pipeline) getPipeline()).getResultRows().addAll(data.rows);
-
+      handOverRows();
       setOutputDone();
       return false;
     }
 
-    // Add all rows to rows buffer...
-    data.rows.add(new RowMetaAndData(getInputRowMeta(), r));
-    data.outputRowMeta = getInputRowMeta().clone();
-    meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, 
metadataProvider);
-    putRow(data.outputRowMeta, r); // copy row to possible alternate
-    // rowset(s).
+    if (first) {
+      first = false;
+      data.outputRowMeta = getInputRowMeta().clone();
+      meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, 
metadataProvider);
+      data.resultRowMeta = data.outputRowMeta.clone();
+    }
+
+    // Keep a copy of the row: the transforms after this one get the same 
array and may change it
+    // in place.
+    data.rows.add(new RowMetaAndData(data.resultRowMeta, 
data.resultRowMeta.cloneRow(r)));
+    putRow(data.outputRowMeta, r);
 
     if (checkFeedback(getLinesRead()) && isBasic()) {
       logBasic(BaseMessages.getString(PKG, "RowsToResult.Log.LineNumber") + 
getLinesRead());
@@ -66,6 +70,30 @@ public class RowsToResult extends 
BaseTransform<RowsToResultMeta, RowsToResultDa
     return true;
   }
 
+  /**
+   * The single threaded executor calls processRow() only for the rows waiting 
on the input, so it
+   * never reaches the end of the input there. Hand over the rows of every 
batch instead.
+   */
+  @Override
+  public void batchComplete() throws HopException {
+    handOverRows();
+  }
+
+  /**
+   * Adds the buffered rows to the pipeline's result rows and empties the 
buffer. Several copies, or
+   * several Copy rows to result transforms, add to the same list, so the list 
is locked.
+   */
+  private void handOverRows() {
+    if (data.rows.isEmpty()) {
+      return;
+    }
+    List<RowMetaAndData> resultRows = ((Pipeline) 
getPipeline()).getResultRows();
+    synchronized (resultRows) {
+      resultRows.addAll(data.rows);
+    }
+    data.rows.clear();
+  }
+
   @Override
   public boolean init() {
 
diff --git 
a/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultData.java
 
b/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultData.java
index dbf1f7caeb..e545595cf1 100644
--- 
a/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultData.java
+++ 
b/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultData.java
@@ -26,9 +26,18 @@ import org.apache.hop.pipeline.transform.ITransformData;
 
 @SuppressWarnings("java:S1104")
 public class RowsToResultData extends BaseTransformData implements 
ITransformData {
+  /** Rows buffered by this copy, not yet handed over to the pipeline's result 
rows. */
   public List<RowMetaAndData> rows;
+
+  /** The metadata of the rows passed on to the next transforms, built on the 
first row. */
   public IRowMeta outputRowMeta;
 
+  /**
+   * The metadata stored with every result row, built on the first row. It is 
a separate copy of
+   * {@link #outputRowMeta}: the result rows leave the pipeline, so nothing 
downstream shares it.
+   */
+  public IRowMeta resultRowMeta;
+
   public RowsToResultData() {
     super();
 
diff --git 
a/plugins/transforms/rowstoresult/src/test/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultTest.java
 
b/plugins/transforms/rowstoresult/src/test/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultTest.java
new file mode 100644
index 0000000000..4ba716c6ff
--- /dev/null
+++ 
b/plugins/transforms/rowstoresult/src/test/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultTest.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.rowstoresult;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.RowMetaAndData;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.RowProducer;
+import org.apache.hop.pipeline.SingleThreadedPipelineExecutor;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+/** Copy rows to result in a real pipeline: Injector, then Copy rows to 
result, then Dummy. */
+class RowsToResultTest {
+
+  private static final Duration TIMEOUT = Duration.ofSeconds(60);
+  private static final String INPUT = "input";
+  private static final String TO_RESULT = "to result";
+  private static final String OUTPUT = "output";
+
+  @BeforeAll
+  static void initHop() throws Exception {
+    HopEnvironment.init();
+    PluginRegistry registry = PluginRegistry.getInstance();
+    for (ITransformMeta meta :
+        List.of(new RowsToResultMeta(), new InjectorMeta(), new DummyMeta())) {
+      if (registry.getPluginId(TransformPluginType.class, meta) == null) {
+        registry.registerPluginClass(
+            meta.getClass().getName(), TransformPluginType.class, 
Transform.class);
+      }
+    }
+  }
+
+  /** A transform after Copy rows to result that changes the row in place, as 
If null does. */
+  @Test
+  void laterInPlaceChangeDoesNotReachTheResultRows() {
+    List<RowMetaAndData> resultRows =
+        assertTimeoutPreemptively(
+            TIMEOUT,
+            () -> {
+              Pipeline pipeline = preparePipeline(1, 
PipelineMeta.PipelineType.Normal);
+              pipeline
+                  .getTransform(OUTPUT, 0)
+                  .addRowListener(
+                      new RowAdapter() {
+                        @Override
+                        public void rowReadEvent(IRowMeta rowMeta, Object[] 
row)
+                            throws HopTransformException {
+                          row[1] = "changed downstream";
+                        }
+                      });
+              return runMultiThreaded(pipeline, 3);
+            });
+
+    assertEquals(List.of("value-0", "value-1", "value-2"), names(resultRows));
+  }
+
+  @Test
+  void resultRowsKeepTheFieldsAndShareOneRowMeta() {
+    List<RowMetaAndData> resultRows =
+        assertTimeoutPreemptively(
+            TIMEOUT,
+            () -> runMultiThreaded(preparePipeline(1, 
PipelineMeta.PipelineType.Normal), 3));
+
+    assertEquals(3, resultRows.size());
+    IRowMeta first = resultRows.get(0).getRowMeta();
+    assertEquals(List.of("id", "name"), List.of(first.getFieldNames()));
+    for (RowMetaAndData row : resultRows) {
+      assertSame(first, row.getRowMeta());
+    }
+  }
+
+  /**
+   * Several copies hand over their rows when their input ends, at about the 
same moment, into one
+   * list. Repeated because a lost update depends on thread timing.
+   */
+  @Test
+  void severalCopiesLoseNoRows() {
+    int copies = 4;
+    int nrRows = 40_000;
+    for (int attempt = 0; attempt < 10; attempt++) {
+      List<RowMetaAndData> resultRows =
+          assertTimeoutPreemptively(
+              TIMEOUT,
+              () ->
+                  runMultiThreaded(
+                      preparePipeline(copies, 
PipelineMeta.PipelineType.Normal), nrRows));
+
+      assertEquals(nrRows, resultRows.size(), "attempt " + attempt);
+      boolean[] seen = new boolean[nrRows];
+      for (RowMetaAndData row : resultRows) {
+        int id = (int) (long) (Long) row.getData()[0];
+        assertTrue(!seen[id], "row " + id + " added twice");
+        seen[id] = true;
+      }
+    }
+  }
+
+  /**
+   * The single threaded executor calls processRow() only for the rows waiting 
on the input, then
+   * batchComplete(). The rows of every batch reach the result rows, once.
+   */
+  @Test
+  void singleThreadedExecutorReturnsEveryRowOnce() {
+    List<RowMetaAndData> resultRows =
+        assertTimeoutPreemptively(
+            TIMEOUT,
+            () -> {
+              Pipeline pipeline = preparePipeline(1, 
PipelineMeta.PipelineType.SingleThreaded);
+              RowProducer producer = pipeline.addRowProducer(INPUT, 0);
+              pipeline.startThreads();
+
+              SingleThreadedPipelineExecutor executor =
+                  new SingleThreadedPipelineExecutor(pipeline);
+              assertTrue(executor.init());
+              try {
+                putRows(producer, 0, 3);
+                executor.oneIteration();
+                putRows(producer, 3, 2);
+                executor.oneIteration();
+                producer.finished();
+                executor.oneIteration();
+              } finally {
+                executor.dispose();
+              }
+              assertEquals(0, pipeline.getTransform(TO_RESULT, 0).getErrors());
+              return pipeline.getResultRows();
+            });
+
+    assertEquals(List.of("value-0", "value-1", "value-2", "value-3", 
"value-4"), names(resultRows));
+  }
+
+  @Test
+  void emptyInputGivesNoResultRows() {
+    List<RowMetaAndData> resultRows =
+        assertTimeoutPreemptively(
+            TIMEOUT,
+            () -> runMultiThreaded(preparePipeline(1, 
PipelineMeta.PipelineType.Normal), 0));
+
+    assertTrue(resultRows.isEmpty());
+  }
+
+  private static Pipeline preparePipeline(int copies, 
PipelineMeta.PipelineType type)
+      throws Exception {
+    PipelineMeta pipelineMeta = new PipelineMeta();
+    pipelineMeta.setName("rows-to-result");
+
+    TransformMeta input = addTransform(pipelineMeta, INPUT, new 
InjectorMeta());
+    TransformMeta toResult = addTransform(pipelineMeta, TO_RESULT, new 
RowsToResultMeta());
+    toResult.setCopies(copies);
+    TransformMeta output = addTransform(pipelineMeta, OUTPUT, new DummyMeta());
+    pipelineMeta.addPipelineHop(new PipelineHopMeta(input, toResult));
+    pipelineMeta.addPipelineHop(new PipelineHopMeta(toResult, output));
+
+    Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
+    pipeline.setPipelineType(type);
+    pipeline.prepareExecution();
+    return pipeline;
+  }
+
+  private static List<RowMetaAndData> runMultiThreaded(Pipeline pipeline, int 
nrRows)
+      throws Exception {
+    RowProducer producer = pipeline.addRowProducer(INPUT, 0);
+    pipeline.startThreads();
+    putRows(producer, 0, nrRows);
+    producer.finished();
+    pipeline.waitUntilFinished();
+    assertEquals(0, pipeline.getErrors());
+    return pipeline.getResultRows();
+  }
+
+  private static void putRows(RowProducer producer, int firstId, int nrRows) {
+    IRowMeta rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaInteger("id"));
+    rowMeta.addValueMeta(new ValueMetaString("name"));
+    for (int id = firstId; id < firstId + nrRows; id++) {
+      producer.putRow(rowMeta, new Object[] {(long) id, "value-" + id});
+    }
+  }
+
+  private static List<String> names(List<RowMetaAndData> rows) {
+    List<String> names = new ArrayList<>();
+    for (RowMetaAndData row : rows) {
+      names.add((String) row.getData()[1]);
+    }
+    return names;
+  }
+
+  private static TransformMeta addTransform(
+      PipelineMeta pipelineMeta, String name, ITransformMeta meta) {
+    String pluginId = 
PluginRegistry.getInstance().getPluginId(TransformPluginType.class, meta);
+    TransformMeta transformMeta = new TransformMeta(pluginId, name, meta);
+    pipelineMeta.addTransform(transformMeta);
+    return transformMeta;
+  }
+}

Reply via email to