This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 3674b5e70a Improve copy rows to result, fixes #9328 (#9331)
3674b5e70a is described below
commit 3674b5e70a50083fffcb1363a6f4af2406b285e7
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 20:31:57 2026 +0100
Improve copy rows to result, fixes #9328 (#9331)
---
.../0004-rows-to-result-validation.hpl | 282 +++++++++++++++++++++
.../single_threaded/0004-rows-to-result.hpl | 122 +++++++++
.../single_threaded/main-0004-rows-to-result.hwf | 138 ++++++++++
.../transforms/rowstoresult/RowsToResult.java | 46 +++-
.../transforms/rowstoresult/RowsToResultData.java | 9 +
.../transforms/rowstoresult/RowsToResultTest.java | 235 +++++++++++++++++
6 files changed, 823 insertions(+), 9 deletions(-)
diff --git
a/integration-tests/single_threaded/0004-rows-to-result-validation.hpl
b/integration-tests/single_threaded/0004-rows-to-result-validation.hpl
new file mode 100644
index 0000000000..341f7b4e4f
--- /dev/null
+++ b/integration-tests/single_threaded/0004-rows-to-result-validation.hpl
@@ -0,0 +1,282 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0004-rows-to-result-validation</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Verify the result rows of 0004-rows-to-result, which ran on
the single threaded engine</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/06 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/06 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Expected</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Output of the test</from>
+ <to>Sort output</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Sort output</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Merge rows (diff)</from>
+ <to>Not identical</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Not identical</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Expected</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>alpha</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>bravo</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>charlie</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>delta</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>echo</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Output of the test</name>
+ <type>RowsFromResult</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>name</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sort output</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <sort_prefix>out</sort_prefix>
+ <sort_size>1000000</sort_size>
+ <free_memory/>
+ <unique_rows>N</unique_rows>
+ <compress>N</compress>
+ <fields>
+ <field>
+ <name>id</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <name>name</name>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Merge rows (diff)</name>
+ <type>MergeRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <keys>
+ <key>id</key>
+ <key>name</key>
+ </keys>
+ <values>
+ <value>id</value>
+ <value>name</value>
+ </values>
+ <flag_field>flagfield</flag_field>
+ <reference>Expected</reference>
+ <compare>Sort output</compare>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Not identical</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to/>
+ <send_false_to/>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <leftvalue>flagfield</leftvalue>
+ <function><></function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>identical</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ <mask/>
+ </value>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <message>0004-rows-to-result: the result rows of Copy rows to result on
the single threaded engine are not the expected ones</message>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>736</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/single_threaded/0004-rows-to-result.hpl
b/integration-tests/single_threaded/0004-rows-to-result.hpl
new file mode 100644
index 0000000000..7007ccba15
--- /dev/null
+++ b/integration-tests/single_threaded/0004-rows-to-result.hpl
@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0004-rows-to-result</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Copy rows to result on the single threaded engine: the rows
must reach the result (#9328)</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/06 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/06 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>rows</from>
+ <to>Copy rows to result</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>rows</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>alpha</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>bravo</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>charlie</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>delta</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>echo</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Copy rows to result</name>
+ <type>RowsToResult</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/single_threaded/main-0004-rows-to-result.hwf
b/integration-tests/single_threaded/main-0004-rows-to-result.hwf
new file mode 100644
index 0000000000..5c185b2bef
--- /dev/null
+++ b/integration-tests/single_threaded/main-0004-rows-to-result.hwf
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0004-rows-to-result</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/06 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/06 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <start>Y</start>
+ <dummy>N</dummy>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0004-rows-to-result.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0004-rows-to-result.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>single-threaded</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0004-rows-to-result-validation.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0004-rows-to-result-validation.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>608</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0004-rows-to-result.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0004-rows-to-result.hpl</from>
+ <to>0004-rows-to-result-validation.hpl</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResult.java
b/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResult.java
index 2aa59ac694..cec044221c 100644
---
a/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResult.java
+++
b/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResult.java
@@ -17,6 +17,7 @@
package org.apache.hop.pipeline.transforms.rowstoresult;
+import java.util.List;
import org.apache.hop.core.RowMetaAndData;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.i18n.BaseMessages;
@@ -45,19 +46,22 @@ public class RowsToResult extends
BaseTransform<RowsToResultMeta, RowsToResultDa
Object[] r = getRow(); // get row, set busy!
if (r == null) { // no more input to be expected...
-
- ((Pipeline) getPipeline()).getResultRows().addAll(data.rows);
-
+ handOverRows();
setOutputDone();
return false;
}
- // Add all rows to rows buffer...
- data.rows.add(new RowMetaAndData(getInputRowMeta(), r));
- data.outputRowMeta = getInputRowMeta().clone();
- meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
- putRow(data.outputRowMeta, r); // copy row to possible alternate
- // rowset(s).
+ if (first) {
+ first = false;
+ data.outputRowMeta = getInputRowMeta().clone();
+ meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
+ data.resultRowMeta = data.outputRowMeta.clone();
+ }
+
+ // Keep a copy of the row: the transforms after this one get the same
array and may change it
+ // in place.
+ data.rows.add(new RowMetaAndData(data.resultRowMeta,
data.resultRowMeta.cloneRow(r)));
+ putRow(data.outputRowMeta, r);
if (checkFeedback(getLinesRead()) && isBasic()) {
logBasic(BaseMessages.getString(PKG, "RowsToResult.Log.LineNumber") +
getLinesRead());
@@ -66,6 +70,30 @@ public class RowsToResult extends
BaseTransform<RowsToResultMeta, RowsToResultDa
return true;
}
+ /**
+ * The single threaded executor calls processRow() only for the rows waiting
on the input, so it
+ * never reaches the end of the input there. Hand over the rows of every
batch instead.
+ */
+ @Override
+ public void batchComplete() throws HopException {
+ handOverRows();
+ }
+
+ /**
+ * Adds the buffered rows to the pipeline's result rows and empties the
buffer. Several copies, or
+ * several Copy rows to result transforms, add to the same list, so the list
is locked.
+ */
+ private void handOverRows() {
+ if (data.rows.isEmpty()) {
+ return;
+ }
+ List<RowMetaAndData> resultRows = ((Pipeline)
getPipeline()).getResultRows();
+ synchronized (resultRows) {
+ resultRows.addAll(data.rows);
+ }
+ data.rows.clear();
+ }
+
@Override
public boolean init() {
diff --git
a/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultData.java
b/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultData.java
index dbf1f7caeb..e545595cf1 100644
---
a/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultData.java
+++
b/plugins/transforms/rowstoresult/src/main/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultData.java
@@ -26,9 +26,18 @@ import org.apache.hop.pipeline.transform.ITransformData;
@SuppressWarnings("java:S1104")
public class RowsToResultData extends BaseTransformData implements
ITransformData {
+ /** Rows buffered by this copy, not yet handed over to the pipeline's result
rows. */
public List<RowMetaAndData> rows;
+
+ /** The metadata of the rows passed on to the next transforms, built on the
first row. */
public IRowMeta outputRowMeta;
+ /**
+ * The metadata stored with every result row, built on the first row. It is
a separate copy of
+ * {@link #outputRowMeta}: the result rows leave the pipeline, so nothing
downstream shares it.
+ */
+ public IRowMeta resultRowMeta;
+
public RowsToResultData() {
super();
diff --git
a/plugins/transforms/rowstoresult/src/test/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultTest.java
b/plugins/transforms/rowstoresult/src/test/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultTest.java
new file mode 100644
index 0000000000..4ba716c6ff
--- /dev/null
+++
b/plugins/transforms/rowstoresult/src/test/java/org/apache/hop/pipeline/transforms/rowstoresult/RowsToResultTest.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.rowstoresult;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.RowMetaAndData;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.RowProducer;
+import org.apache.hop.pipeline.SingleThreadedPipelineExecutor;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+/** Copy rows to result in a real pipeline: Injector, then Copy rows to
result, then Dummy. */
+class RowsToResultTest {
+
+ private static final Duration TIMEOUT = Duration.ofSeconds(60);
+ private static final String INPUT = "input";
+ private static final String TO_RESULT = "to result";
+ private static final String OUTPUT = "output";
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopEnvironment.init();
+ PluginRegistry registry = PluginRegistry.getInstance();
+ for (ITransformMeta meta :
+ List.of(new RowsToResultMeta(), new InjectorMeta(), new DummyMeta())) {
+ if (registry.getPluginId(TransformPluginType.class, meta) == null) {
+ registry.registerPluginClass(
+ meta.getClass().getName(), TransformPluginType.class,
Transform.class);
+ }
+ }
+ }
+
+ /** A transform after Copy rows to result that changes the row in place, as
If null does. */
+ @Test
+ void laterInPlaceChangeDoesNotReachTheResultRows() {
+ List<RowMetaAndData> resultRows =
+ assertTimeoutPreemptively(
+ TIMEOUT,
+ () -> {
+ Pipeline pipeline = preparePipeline(1,
PipelineMeta.PipelineType.Normal);
+ pipeline
+ .getTransform(OUTPUT, 0)
+ .addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowReadEvent(IRowMeta rowMeta, Object[]
row)
+ throws HopTransformException {
+ row[1] = "changed downstream";
+ }
+ });
+ return runMultiThreaded(pipeline, 3);
+ });
+
+ assertEquals(List.of("value-0", "value-1", "value-2"), names(resultRows));
+ }
+
+ @Test
+ void resultRowsKeepTheFieldsAndShareOneRowMeta() {
+ List<RowMetaAndData> resultRows =
+ assertTimeoutPreemptively(
+ TIMEOUT,
+ () -> runMultiThreaded(preparePipeline(1,
PipelineMeta.PipelineType.Normal), 3));
+
+ assertEquals(3, resultRows.size());
+ IRowMeta first = resultRows.get(0).getRowMeta();
+ assertEquals(List.of("id", "name"), List.of(first.getFieldNames()));
+ for (RowMetaAndData row : resultRows) {
+ assertSame(first, row.getRowMeta());
+ }
+ }
+
+ /**
+ * Several copies hand over their rows when their input ends, at about the
same moment, into one
+ * list. Repeated because a lost update depends on thread timing.
+ */
+ @Test
+ void severalCopiesLoseNoRows() {
+ int copies = 4;
+ int nrRows = 40_000;
+ for (int attempt = 0; attempt < 10; attempt++) {
+ List<RowMetaAndData> resultRows =
+ assertTimeoutPreemptively(
+ TIMEOUT,
+ () ->
+ runMultiThreaded(
+ preparePipeline(copies,
PipelineMeta.PipelineType.Normal), nrRows));
+
+ assertEquals(nrRows, resultRows.size(), "attempt " + attempt);
+ boolean[] seen = new boolean[nrRows];
+ for (RowMetaAndData row : resultRows) {
+ int id = (int) (long) (Long) row.getData()[0];
+ assertTrue(!seen[id], "row " + id + " added twice");
+ seen[id] = true;
+ }
+ }
+ }
+
+ /**
+ * The single threaded executor calls processRow() only for the rows waiting
on the input, then
+ * batchComplete(). The rows of every batch reach the result rows, once.
+ */
+ @Test
+ void singleThreadedExecutorReturnsEveryRowOnce() {
+ List<RowMetaAndData> resultRows =
+ assertTimeoutPreemptively(
+ TIMEOUT,
+ () -> {
+ Pipeline pipeline = preparePipeline(1,
PipelineMeta.PipelineType.SingleThreaded);
+ RowProducer producer = pipeline.addRowProducer(INPUT, 0);
+ pipeline.startThreads();
+
+ SingleThreadedPipelineExecutor executor =
+ new SingleThreadedPipelineExecutor(pipeline);
+ assertTrue(executor.init());
+ try {
+ putRows(producer, 0, 3);
+ executor.oneIteration();
+ putRows(producer, 3, 2);
+ executor.oneIteration();
+ producer.finished();
+ executor.oneIteration();
+ } finally {
+ executor.dispose();
+ }
+ assertEquals(0, pipeline.getTransform(TO_RESULT, 0).getErrors());
+ return pipeline.getResultRows();
+ });
+
+ assertEquals(List.of("value-0", "value-1", "value-2", "value-3",
"value-4"), names(resultRows));
+ }
+
+ @Test
+ void emptyInputGivesNoResultRows() {
+ List<RowMetaAndData> resultRows =
+ assertTimeoutPreemptively(
+ TIMEOUT,
+ () -> runMultiThreaded(preparePipeline(1,
PipelineMeta.PipelineType.Normal), 0));
+
+ assertTrue(resultRows.isEmpty());
+ }
+
+ private static Pipeline preparePipeline(int copies,
PipelineMeta.PipelineType type)
+ throws Exception {
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("rows-to-result");
+
+ TransformMeta input = addTransform(pipelineMeta, INPUT, new
InjectorMeta());
+ TransformMeta toResult = addTransform(pipelineMeta, TO_RESULT, new
RowsToResultMeta());
+ toResult.setCopies(copies);
+ TransformMeta output = addTransform(pipelineMeta, OUTPUT, new DummyMeta());
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(input, toResult));
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(toResult, output));
+
+ Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
+ pipeline.setPipelineType(type);
+ pipeline.prepareExecution();
+ return pipeline;
+ }
+
+ private static List<RowMetaAndData> runMultiThreaded(Pipeline pipeline, int
nrRows)
+ throws Exception {
+ RowProducer producer = pipeline.addRowProducer(INPUT, 0);
+ pipeline.startThreads();
+ putRows(producer, 0, nrRows);
+ producer.finished();
+ pipeline.waitUntilFinished();
+ assertEquals(0, pipeline.getErrors());
+ return pipeline.getResultRows();
+ }
+
+ private static void putRows(RowProducer producer, int firstId, int nrRows) {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("id"));
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ for (int id = firstId; id < firstId + nrRows; id++) {
+ producer.putRow(rowMeta, new Object[] {(long) id, "value-" + id});
+ }
+ }
+
+ private static List<String> names(List<RowMetaAndData> rows) {
+ List<String> names = new ArrayList<>();
+ for (RowMetaAndData row : rows) {
+ names.add((String) row.getData()[1]);
+ }
+ return names;
+ }
+
+ private static TransformMeta addTransform(
+ PipelineMeta pipelineMeta, String name, ITransformMeta meta) {
+ String pluginId =
PluginRegistry.getInstance().getPluginId(TransformPluginType.class, meta);
+ TransformMeta transformMeta = new TransformMeta(pluginId, name, meta);
+ pipelineMeta.addTransform(transformMeta);
+ return transformMeta;
+ }
+}