This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 0f1184527f Issue #5660 (#5665)
0f1184527f is described below
commit 0f1184527f620758127368ae130ad62a457e1e90
Author: Matt Casters <[email protected]>
AuthorDate: Mon Sep 8 16:05:54 2025 +0200
Issue #5660 (#5665)
* Issue #5660 (plus code cleanup and integration tests, MDI fixes, doc
update)
* Issue #5660 (dialog code hardening)
* rebase code
---------
Co-authored-by: Matt Casters <[email protected]>
Co-authored-by: Hans Van Akelyen <[email protected]>
---
.../ROOT/pages/pipeline/transforms/mergerows.adoc | 17 ++
integration-tests/mdi/0030-merge-rows-template.hpl | 289 ++++++++++++++++++++
integration-tests/mdi/0030-merge-rows.hpl | 282 ++++++++++++++++++++
.../mdi/datasets/golden-merge-rows.csv | 13 +
integration-tests/mdi/main-0030-merge-rows.hwf | 80 ++++++
.../mdi/metadata/dataset/golden-merge-rows.json | 40 +++
.../metadata/unit-test/0030-merge-rows UNIT.json | 43 +++
.../transforms/0077-merge-rows-diff.hpl | 291 ++++++++++++++++++++
integration-tests/transforms/0077-merge-rows.hpl | 292 +++++++++++++++++++++
.../transforms/datasets/golden-merge-rows-diff.csv | 13 +
.../transforms/datasets/golden-merge-rows.csv | 13 +
.../transforms/main-0077-merge-rows.hwf | 83 ++++++
.../metadata/dataset/golden-merge-rows-diff.json | 48 ++++
.../metadata/dataset/golden-merge-rows.json | 40 +++
.../metadata/unit-test/0077-merge-rows UNIT.json | 40 +++
.../unit-test/0077-merge-rows-diff UNIT.json | 48 ++++
.../pipeline/transforms/mergerows/MergeRows.java | 65 +++--
.../transforms/mergerows/MergeRowsDialog.java | 65 +++--
.../transforms/mergerows/MergeRowsMeta.java | 221 +++++-----------
.../mergerows/messages/messages_en_US.properties | 6 +-
.../mergerows/MergeRowsMetaInjectionTest.java | 41 ---
.../transforms/mergerows/MergeRowsMetaTest.java | 67 ++---
.../mergerows/src/test/resources/merge-rows.xml | 30 +++
.../org/apache/hop/ui/hopgui/HopCommandGui.java | 2 +
24 files changed, 1844 insertions(+), 285 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/mergerows.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/mergerows.adoc
index faa49a235d..f8d8b317e6 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/mergerows.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/mergerows.adoc
@@ -76,7 +76,24 @@ In the subsequent transform, you can use the flag field
generated by **Merge row
|Reference rows origin|Specify the transform that produces the reference rows.
It's a Stream with original rows (rows that you want to compare the new rows
to).
|Compare rows origin|Specify the transform that produces the compare rows.
It's a Stream with new rows
|Flag fieldname|Specify the name of the flag field on the output stream.
+|Difference fieldname|Specify the name of the field to contain the difference
in case the status is **changed**.
+The difference will be in the form of a "changes" array in JSON format.
|Keys to match|Specify fields containing the keys on which to match. Click
"Get key fields" to insert all of the fields from the reference rows
|Values to compare|Specify fields contaning the values to compare. Click "Get
value fields" to insert all of the fields from the compare rows.
Key fields do not need to be repeated here.
|===
+
+== Differences JSON example
+
+If you specify a field name for the differences in JSON, you will changes
appear when a change was detected:
+
+[source,json]
+----
+{
+ "changes" : [
+ { "field1" : { "from" : "aaa", "to" : "bbb" }},
+ { "field2" : { "from" : 111, "to" : 222 }}
+ ...
+ ]
+}
+----
diff --git a/integration-tests/mdi/0030-merge-rows-template.hpl
b/integration-tests/mdi/0030-merge-rows-template.hpl
new file mode 100644
index 0000000000..be58d08e7e
--- /dev/null
+++ b/integration-tests/mdi/0030-merge-rows-template.hpl
@@ -0,0 +1,289 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0030-merge-rows-template</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2025/09/02 11:26:20.192</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/09/02 11:26:20.192</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>source1</from>
+ <to>Merge rows</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>source2</from>
+ <to>Merge rows</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Merge rows</from>
+ <to>OUTPUT</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Merge rows</name>
+ <type>MergeRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <keys>
+ </keys>
+ <values>
+ </values>
+ <flag_field/>
+ <reference>source1</reference>
+ <compare>source2</compare>
+ <compare>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>source1</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>James</item>
+ <item>3.6</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>Marie</item>
+ <item>8.3</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>Michael</item>
+ <item>9.1</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>Patrcia</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>Robert</item>
+ <item>5.0</item>
+ </line>
+ <line>
+ <item>6</item>
+ <item>Linda</item>
+ <item>1.6</item>
+ </line>
+ <line>
+ <item>7</item>
+ <item>David</item>
+ <item>7.2</item>
+ </line>
+ <line>
+ <item>8</item>
+ <item>Elizabeth</item>
+ <item>6.9</item>
+ </line>
+ <line>
+ <item>9</item>
+ <item>Willian</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>10</item>
+ <item>Barbara</item>
+ <item>0.5</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>score</name>
+ <format>#.#</format>
+ <type>Number</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>source2</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>James</item>
+ <item>3.6</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>Michael</item>
+ <item>9.1</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>Patricia</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>Robert</item>
+ <item>5.0</item>
+ </line>
+ <line>
+ <item>7</item>
+ <item>David</item>
+ <item>7.2</item>
+ </line>
+ <line>
+ <item>8</item>
+ <item>Elizabeth</item>
+ <item>6.9</item>
+ </line>
+ <line>
+ <item>9</item>
+ <item>William</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>10</item>
+ <item>Barbara</item>
+ <item>0.5</item>
+ </line>
+ <line>
+ <item>11</item>
+ <item>Richard</item>
+ <item>9.6</item>
+ </line>
+ <line>
+ <item>12</item>
+ <item>Susan</item>
+ <item>5.4</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>score</name>
+ <format>#.#</format>
+ <type>Number</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>OUTPUT</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>432</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/mdi/0030-merge-rows.hpl
b/integration-tests/mdi/0030-merge-rows.hpl
new file mode 100644
index 0000000000..b14f42124c
--- /dev/null
+++ b/integration-tests/mdi/0030-merge-rows.hpl
@@ -0,0 +1,282 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0030-merge-rows</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2025/09/02 11:28:05.214</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/09/02 11:28:05.214</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>metadata</from>
+ <to>ETL metadata injection</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>keys</from>
+ <to>ETL metadata injection</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>values</from>
+ <to>ETL metadata injection</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>ETL metadata injection</from>
+ <to>OUTPUT</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>ETL metadata injection</name>
+ <type>MetaInject</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <filename>${PROJECT_HOME}/0030-merge-rows-template.hpl</filename>
+ <run_configuration>local</run_configuration>
+ <source_transform>OUTPUT</source_transform>
+ <source_output_fields>
+ <source_output_field>
+ <source_output_field_name>id</source_output_field_name>
+ <source_output_field_type>Integer</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>name</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>score</source_output_field_name>
+ <source_output_field_type>Number</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>flagfield</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ </source_output_fields>
+ <target_file/>
+ <create_parent_folder>Y</create_parent_folder>
+ <no_execution>N</no_execution>
+ <stream_source_transform/>
+ <stream_target_transform/>
+ <mappings>
+ <mapping>
+ <target_transform_name>Merge rows</target_transform_name>
+ <target_attribute_key>VALUE_FIELD</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>values</source_transform>
+ <source_field>value</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>Merge rows</target_transform_name>
+ <target_attribute_key>KEY_FIELD</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>keys</source_transform>
+ <source_field>key</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>Merge rows</target_transform_name>
+ <target_attribute_key>FLAG_FIELD</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>metadata</source_transform>
+ <source_field>flagfield</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>Merge rows</target_transform_name>
+ <target_attribute_key>COMPARE_TRANSFORM</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>metadata</source_transform>
+ <source_field>compare</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>Merge rows</target_transform_name>
+ <target_attribute_key>REFERENCE_TRANSFORM</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>metadata</source_transform>
+ <source_field>reference</source_field>
+ </mapping>
+ </mappings>
+ <attributes/>
+ <GUI>
+ <xloc>304</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>OUTPUT</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>keys</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>id</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>key</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>metadata</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>source1</item>
+ <item>source2</item>
+ <item>flagfield</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>reference</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>compare</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>flagfield</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>values</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>name</item>
+ </line>
+ <line>
+ <item>score</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>value</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>240</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/mdi/datasets/golden-merge-rows.csv
b/integration-tests/mdi/datasets/golden-merge-rows.csv
new file mode 100644
index 0000000000..e6b5b37c5e
--- /dev/null
+++ b/integration-tests/mdi/datasets/golden-merge-rows.csv
@@ -0,0 +1,13 @@
+id,name,score,flagfield
+1,James,3.6,identical
+2,Marie,8.3,deleted
+3,Michael,9.1,identical
+4,Patricia,4.2,changed
+5,Robert,5.0,identical
+6,Linda,1.6,deleted
+7,David,7.2,identical
+8,Elizabeth,6.9,identical
+9,William,4.2,changed
+10,Barbara,0.5,identical
+11,Richard,9.6,new
+12,Susan,5.4,new
diff --git a/integration-tests/mdi/main-0030-merge-rows.hwf
b/integration-tests/mdi/main-0030-merge-rows.hwf
new file mode 100644
index 0000000000..e02a2ae162
--- /dev/null
+++ b/integration-tests/mdi/main-0030-merge-rows.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0030-merge-rows</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2025/09/02 11:24:45.914</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/09/02 11:24:45.914</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>80</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0030-merge-rows UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/mdi/metadata/dataset/golden-merge-rows.json
b/integration-tests/mdi/metadata/dataset/golden-merge-rows.json
new file mode 100644
index 0000000000..a0c11e533b
--- /dev/null
+++ b/integration-tests/mdi/metadata/dataset/golden-merge-rows.json
@@ -0,0 +1,40 @@
+{
+ "base_filename": "golden-merge-rows.csv",
+ "name": "golden-merge-rows",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "score",
+ "field_format": "####0.0#########;-####0.0#########"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "flagfield",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git a/integration-tests/mdi/metadata/unit-test/0030-merge-rows UNIT.json
b/integration-tests/mdi/metadata/unit-test/0030-merge-rows UNIT.json
new file mode 100644
index 0000000000..144146a4af
--- /dev/null
+++ b/integration-tests/mdi/metadata/unit-test/0030-merge-rows UNIT.json
@@ -0,0 +1,43 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ },
+ {
+ "transform_field": "score",
+ "data_set_field": "score"
+ },
+ {
+ "transform_field": "flagfield",
+ "data_set_field": "flagfield"
+ }
+ ],
+ "field_order": [
+ "id",
+ "name",
+ "score",
+ "flagfield"
+ ],
+ "data_set_name": "golden-merge-rows",
+ "transform_name": "OUTPUT"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0030-merge-rows UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0030-merge-rows.hpl"
+}
\ No newline at end of file
diff --git a/integration-tests/transforms/0077-merge-rows-diff.hpl
b/integration-tests/transforms/0077-merge-rows-diff.hpl
new file mode 100644
index 0000000000..834df0f156
--- /dev/null
+++ b/integration-tests/transforms/0077-merge-rows-diff.hpl
@@ -0,0 +1,291 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0077-merge-rows-diff</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2025/09/02 10:00:26.808</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/09/02 10:00:26.808</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>source1</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>source2</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Merge rows (diff)</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Merge rows (diff)</name>
+ <type>MergeRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <compare>source2</compare>
+ <diff-field>difference</diff-field>
+ <flag_field>flagfield</flag_field>
+ <keys>
+ <key>id</key>
+ </keys>
+ <reference>source1</reference>
+ <values>
+ <value>name</value>
+ <value>score</value>
+ </values>
+ <attributes/>
+ <GUI>
+ <xloc>224</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>source1</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>James</item>
+ <item>3.6</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>Marie</item>
+ <item>8.3</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>Michael</item>
+ <item>9.1</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>Patrcia</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>Robert</item>
+ <item>5.0</item>
+ </line>
+ <line>
+ <item>6</item>
+ <item>Linda</item>
+ <item>1.6</item>
+ </line>
+ <line>
+ <item>7</item>
+ <item>David</item>
+ <item>7.2</item>
+ </line>
+ <line>
+ <item>8</item>
+ <item>Elizabeth</item>
+ <item>6.9</item>
+ </line>
+ <line>
+ <item>9</item>
+ <item>Willian</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>10</item>
+ <item>Barbara</item>
+ <item>0.5</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>score</name>
+ <format>#.#</format>
+ <type>Number</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>80</xloc>
+ <yloc>32</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>source2</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>James</item>
+ <item>3.6</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>Michael</item>
+ <item>9.1</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>Patricia</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>Robert</item>
+ <item>5.0</item>
+ </line>
+ <line>
+ <item>7</item>
+ <item>David</item>
+ <item>7.2</item>
+ </line>
+ <line>
+ <item>8</item>
+ <item>Elizabeth</item>
+ <item>6.9</item>
+ </line>
+ <line>
+ <item>9</item>
+ <item>William</item>
+ <item>4.3</item>
+ </line>
+ <line>
+ <item>10</item>
+ <item>Barbara</item>
+ <item>0.7</item>
+ </line>
+ <line>
+ <item>11</item>
+ <item>Richard</item>
+ <item>9.6</item>
+ </line>
+ <line>
+ <item>12</item>
+ <item>Susan</item>
+ <item>5.4</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>score</name>
+ <format>#.#</format>
+ <type>Number</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>80</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>400</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0077-merge-rows.hpl
b/integration-tests/transforms/0077-merge-rows.hpl
new file mode 100644
index 0000000000..9c62bd7c66
--- /dev/null
+++ b/integration-tests/transforms/0077-merge-rows.hpl
@@ -0,0 +1,292 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0077-merge-rows</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2025/09/02 10:00:26.808</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/09/02 10:00:26.808</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>source1</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>source2</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Merge rows (diff)</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Merge rows (diff)</name>
+ <type>MergeRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <keys>
+ <key>id</key>
+ </keys>
+ <values>
+ <value>name</value>
+ <value>score</value>
+ </values>
+ <flag_field>flagfield</flag_field>
+ <reference>source1</reference>
+ <compare>source2</compare>
+ <compare>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>source1</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>James</item>
+ <item>3.6</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>Marie</item>
+ <item>8.3</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>Michael</item>
+ <item>9.1</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>Patrcia</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>Robert</item>
+ <item>5.0</item>
+ </line>
+ <line>
+ <item>6</item>
+ <item>Linda</item>
+ <item>1.6</item>
+ </line>
+ <line>
+ <item>7</item>
+ <item>David</item>
+ <item>7.2</item>
+ </line>
+ <line>
+ <item>8</item>
+ <item>Elizabeth</item>
+ <item>6.9</item>
+ </line>
+ <line>
+ <item>9</item>
+ <item>Willian</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>10</item>
+ <item>Barbara</item>
+ <item>0.5</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>score</name>
+ <format>#.#</format>
+ <type>Number</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>source2</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>James</item>
+ <item>3.6</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>Michael</item>
+ <item>9.1</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>Patricia</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>Robert</item>
+ <item>5.0</item>
+ </line>
+ <line>
+ <item>7</item>
+ <item>David</item>
+ <item>7.2</item>
+ </line>
+ <line>
+ <item>8</item>
+ <item>Elizabeth</item>
+ <item>6.9</item>
+ </line>
+ <line>
+ <item>9</item>
+ <item>William</item>
+ <item>4.2</item>
+ </line>
+ <line>
+ <item>10</item>
+ <item>Barbara</item>
+ <item>0.5</item>
+ </line>
+ <line>
+ <item>11</item>
+ <item>Richard</item>
+ <item>9.6</item>
+ </line>
+ <line>
+ <item>12</item>
+ <item>Susan</item>
+ <item>5.4</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>score</name>
+ <format>#.#</format>
+ <type>Number</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/datasets/golden-merge-rows-diff.csv
b/integration-tests/transforms/datasets/golden-merge-rows-diff.csv
new file mode 100644
index 0000000000..b0f1611774
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-merge-rows-diff.csv
@@ -0,0 +1,13 @@
+id,name,score,flagfield,difference
+1,James,3.6,"{""changes"":[]}",identical
+2,Marie,8.3,"{""changes"":[]}",deleted
+3,Michael,9.1,"{""changes"":[]}",identical
+4,Patricia,4.2,"{""changes"":[{""name"":{""from"":""Patrcia"",""to"":""Patricia""}}]}",changed
+5,Robert,5,"{""changes"":[]}",identical
+6,Linda,1.6,"{""changes"":[]}",deleted
+7,David,7.2,"{""changes"":[]}",identical
+8,Elizabeth,6.9,"{""changes"":[]}",identical
+9,William,4.3,"{""changes"":[{""name"":{""from"":""Willian"",""to"":""William""}},{""score"":{""from"":4.2,""to"":4.3}}]}",changed
+10,Barbara,0.7,"{""changes"":[{""score"":{""from"":0.5,""to"":0.7}}]}",changed
+11,Richard,9.6,"{""changes"":[]}",new
+12,Susan,5.4,"{""changes"":[]}",new
diff --git a/integration-tests/transforms/datasets/golden-merge-rows.csv
b/integration-tests/transforms/datasets/golden-merge-rows.csv
new file mode 100644
index 0000000000..b386349627
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-merge-rows.csv
@@ -0,0 +1,13 @@
+id,name,score,flagfield
+1,James,3.6,identical
+2,Marie,8.3,deleted
+3,Michael,9.1,identical
+4,Patricia,4.2,changed
+5,Robert,5,identical
+6,Linda,1.6,deleted
+7,David,7.2,identical
+8,Elizabeth,6.9,identical
+9,William,4.2,changed
+10,Barbara,0.5,identical
+11,Richard,9.6,new
+12,Susan,5.4,new
diff --git a/integration-tests/transforms/main-0077-merge-rows.hwf
b/integration-tests/transforms/main-0077-merge-rows.hwf
new file mode 100644
index 0000000000..3a2878fbad
--- /dev/null
+++ b/integration-tests/transforms/main-0077-merge-rows.hwf
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0077-merge-rows</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2025/09/02 10:00:02.532</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/09/02 10:00:02.532</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>80</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0077-merge-rows UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0077-merge-rows-diff UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>272</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-merge-rows-diff.json
b/integration-tests/transforms/metadata/dataset/golden-merge-rows-diff.json
new file mode 100644
index 0000000000..ca0feaf854
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-merge-rows-diff.json
@@ -0,0 +1,48 @@
+{
+ "base_filename": "golden-merge-rows-diff.csv",
+ "name": "golden-merge-rows-diff",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "score",
+ "field_format": "#.#"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "flagfield",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "difference",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/dataset/golden-merge-rows.json
b/integration-tests/transforms/metadata/dataset/golden-merge-rows.json
new file mode 100644
index 0000000000..9a01d44f42
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-merge-rows.json
@@ -0,0 +1,40 @@
+{
+ "base_filename": "golden-merge-rows.csv",
+ "name": "golden-merge-rows",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "score",
+ "field_format": "#.#"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "flagfield",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git a/integration-tests/transforms/metadata/unit-test/0077-merge-rows
UNIT.json b/integration-tests/transforms/metadata/unit-test/0077-merge-rows
UNIT.json
new file mode 100644
index 0000000000..59f66f05b4
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0077-merge-rows UNIT.json
@@ -0,0 +1,40 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ },
+ {
+ "transform_field": "score",
+ "data_set_field": "score"
+ },
+ {
+ "transform_field": "flagfield",
+ "data_set_field": "flagfield"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "data_set_name": "golden-merge-rows",
+ "transform_name": "validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0077-merge-rows UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0077-merge-rows.hpl"
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0077-merge-rows-diff
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0077-merge-rows-diff UNIT.json
new file mode 100644
index 0000000000..d818cf7416
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0077-merge-rows-diff
UNIT.json
@@ -0,0 +1,48 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ },
+ {
+ "transform_field": "score",
+ "data_set_field": "score"
+ },
+ {
+ "transform_field": "flagfield",
+ "data_set_field": "flagfield"
+ },
+ {
+ "transform_field": "difference",
+ "data_set_field": "difference"
+ }
+ ],
+ "field_order": [
+ "id",
+ "name",
+ "score",
+ "flagfield",
+ "difference"
+ ],
+ "data_set_name": "golden-merge-rows-diff",
+ "transform_name": "validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0077-merge-rows-diff UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0077-merge-rows-diff.hpl"
+}
\ No newline at end of file
diff --git
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRows.java
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRows.java
index 2982a98e12..0d2a10aa3b 100644
---
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRows.java
+++
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRows.java
@@ -19,10 +19,12 @@ package org.apache.hop.pipeline.transforms.mergerows;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopRowException;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowDataUtil;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.i18n.BaseMessages;
@@ -32,6 +34,8 @@ import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.ITransform;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transform.stream.IStream;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
/**
* Merge rows from 2 sorted streams to detect changes. Use this as feed for a
dimension in case you
@@ -81,15 +85,15 @@ public class MergeRows extends BaseTransform<MergeRowsMeta,
MergeRowsData> {
if (data.one != null) {
// Find the key indexes:
- data.keyNrs = new int[meta.getKeyFields().length];
+ data.keyNrs = new int[meta.getKeyFields().size()];
for (int i = 0; i < data.keyNrs.length; i++) {
- data.keyNrs[i] =
data.oneRowSet.getRowMeta().indexOfValue(meta.getKeyFields()[i]);
+ data.keyNrs[i] =
data.oneRowSet.getRowMeta().indexOfValue(meta.getKeyFields().get(i));
if (data.keyNrs[i] < 0) {
String message =
BaseMessages.getString(
PKG,
"MergeRows.Exception.UnableToFindFieldInReferenceStream",
- meta.getKeyFields()[i]);
+ meta.getKeyFields().get(i));
logError(message);
throw new HopTransformException(message);
}
@@ -97,15 +101,15 @@ public class MergeRows extends
BaseTransform<MergeRowsMeta, MergeRowsData> {
}
if (data.two != null) {
- data.valueNrs = new int[meta.getValueFields().length];
+ data.valueNrs = new int[meta.getValueFields().size()];
for (int i = 0; i < data.valueNrs.length; i++) {
- data.valueNrs[i] =
data.twoRowSet.getRowMeta().indexOfValue(meta.getValueFields()[i]);
+ data.valueNrs[i] =
data.twoRowSet.getRowMeta().indexOfValue(meta.getValueFields().get(i));
if (data.valueNrs[i] < 0) {
String message =
BaseMessages.getString(
PKG,
"MergeRows.Exception.UnableToFindFieldInReferenceStream",
- meta.getValueFields()[i]);
+ meta.getValueFields().get(i));
logError(message);
throw new HopTransformException(message);
}
@@ -146,20 +150,19 @@ public class MergeRows extends
BaseTransform<MergeRowsMeta, MergeRowsData> {
}
Object[] outputRow;
- int outputIndex;
- String flagField = null;
+ String flagField;
+ String differenceJson = "{\"changes\":[]}";
+ boolean getDifference = StringUtils.isNotEmpty(meta.getDiffJsonField());
if (data.one == null && data.two != null) { // Record 2 is flagged as new!
outputRow = data.two;
- outputIndex = data.twoRowSet.getRowMeta().size();
flagField = VALUE_NEW;
// Also get a next row from compare rowset...
data.two = getRowFrom(data.twoRowSet);
} else if (data.one != null && data.two == null) { // Record 1 is flagged
as deleted!
outputRow = data.one;
- outputIndex = data.oneRowSet.getRowMeta().size();
flagField = VALUE_DELETED;
// Also get a next row from reference rowset...
@@ -169,16 +172,40 @@ public class MergeRows extends
BaseTransform<MergeRowsMeta, MergeRowsData> {
int compare = data.oneRowSet.getRowMeta().compare(data.one, data.two,
data.keyNrs);
if (compare == 0) { // The Key matches, we CAN compare the two rows...
- int compareValues = data.oneRowSet.getRowMeta().compare(data.one,
data.two, data.valueNrs);
+ int compareValues = 0;
+ if (getDifference) {
+ JSONObject j = new JSONObject();
+ JSONArray jChanges = new JSONArray();
+ j.put("changes", jChanges);
+ for (int valueNr : data.valueNrs) {
+ IValueMeta valueMeta =
data.oneRowSet.getRowMeta().getValueMeta(valueNr);
+ Object refData = data.one[valueNr];
+ Object cmpData = data.two[valueNr];
+ int compareValue = valueMeta.compare(refData, cmpData);
+ if (compareValue != 0) {
+ JSONObject jChange = new JSONObject();
+ jChanges.add(jChange);
+ JSONObject jField = new JSONObject();
+ jChange.put(valueMeta.getName(), jField);
+ jField.put("from", refData);
+ jField.put("to", cmpData);
+ compareValues = compareValue;
+ }
+ if (compareValues != 0) {
+ differenceJson = j.toJSONString();
+ }
+ }
+ } else {
+ compareValues = data.oneRowSet.getRowMeta().compare(data.one,
data.two, data.valueNrs);
+ }
+
if (compareValues == 0) {
outputRow = data.two; // documented behavior: use the comparison
stream
- outputIndex = data.twoRowSet.getRowMeta().size();
flagField = VALUE_IDENTICAL;
} else {
// Return the compare (most recent) row
//
outputRow = data.two;
- outputIndex = data.twoRowSet.getRowMeta().size();
flagField = VALUE_CHANGED;
}
@@ -189,13 +216,11 @@ public class MergeRows extends
BaseTransform<MergeRowsMeta, MergeRowsData> {
if (compare < 0) { // one < two
outputRow = data.one;
- outputIndex = data.oneRowSet.getRowMeta().size();
flagField = VALUE_DELETED;
data.one = getRowFrom(data.oneRowSet);
} else {
outputRow = data.two;
- outputIndex = data.twoRowSet.getRowMeta().size();
flagField = VALUE_NEW;
data.two = getRowFrom(data.twoRowSet);
@@ -203,8 +228,16 @@ public class MergeRows extends
BaseTransform<MergeRowsMeta, MergeRowsData> {
}
}
+ // Optionally add the difference JSON field
+ //
+ outputRow = RowDataUtil.resizeArray(outputRow, data.outputRowMeta.size());
+ if (getDifference && differenceJson != null) {
+ outputRow[data.outputRowMeta.size() - 2] = differenceJson;
+ }
+ outputRow[data.outputRowMeta.size() - 1] = flagField;
+
// send the row to the next transforms...
- putRow(data.outputRowMeta, RowDataUtil.addValueData(outputRow,
outputIndex, flagField));
+ putRow(data.outputRowMeta, outputRow);
if (checkFeedback(getLinesRead()) && isBasic()) {
logBasic(BaseMessages.getString(PKG, "MergeRows.LineNumber") +
getLinesRead());
diff --git
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsDialog.java
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsDialog.java
index d9134b9904..bcd41348ec 100644
---
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsDialog.java
+++
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsDialog.java
@@ -57,6 +57,7 @@ public class MergeRowsDialog extends BaseTransformDialog {
private CCombo wCompare;
private Text wFlagField;
+ private Text wDiffField;
private TableView wKeys;
@@ -200,15 +201,15 @@ public class MergeRowsDialog extends BaseTransformDialog {
fdCompare.right = new FormAttachment(100, 0);
wCompare.setLayoutData(fdCompare);
- // TransformName line
- Label wlFlagfield = new Label(shell, SWT.RIGHT);
- wlFlagfield.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.FlagField.Label"));
- PropsUi.setLook(wlFlagfield);
+ // The flag field line
+ Label wlFlagField = new Label(shell, SWT.RIGHT);
+ wlFlagField.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.FlagField.Label"));
+ PropsUi.setLook(wlFlagField);
FormData fdlFlagfield = new FormData();
fdlFlagfield.left = new FormAttachment(0, 0);
fdlFlagfield.right = new FormAttachment(middle, -margin);
fdlFlagfield.top = new FormAttachment(wCompare, margin);
- wlFlagfield.setLayoutData(fdlFlagfield);
+ wlFlagField.setLayoutData(fdlFlagfield);
wFlagField = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
PropsUi.setLook(wFlagField);
wFlagField.addModifyListener(lsMod);
@@ -218,16 +219,34 @@ public class MergeRowsDialog extends BaseTransformDialog {
fdFlagfield.right = new FormAttachment(100, 0);
wFlagField.setLayoutData(fdFlagfield);
+ // The flag field line
+ Label wlDiffField = new Label(shell, SWT.RIGHT);
+ wlDiffField.setText(BaseMessages.getString(PKG,
"MergeRowsDialog.DiffField.Label"));
+ PropsUi.setLook(wlDiffField);
+ FormData fdlDifffield = new FormData();
+ fdlDifffield.left = new FormAttachment(0, 0);
+ fdlDifffield.right = new FormAttachment(middle, -margin);
+ fdlDifffield.top = new FormAttachment(wFlagField, margin);
+ wlDiffField.setLayoutData(fdlDifffield);
+ wDiffField = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wDiffField);
+ wDiffField.addModifyListener(lsMod);
+ FormData fdDifffield = new FormData();
+ fdDifffield.top = new FormAttachment(wFlagField, margin);
+ fdDifffield.left = new FormAttachment(middle, 0);
+ fdDifffield.right = new FormAttachment(100, 0);
+ wDiffField.setLayoutData(fdDifffield);
+
// THE KEYS TO MATCH...
Label wlKeys = new Label(shell, SWT.NONE);
wlKeys.setText(BaseMessages.getString(PKG, "MergeRowsDialog.Keys.Label"));
PropsUi.setLook(wlKeys);
FormData fdlKeys = new FormData();
fdlKeys.left = new FormAttachment(0, 0);
- fdlKeys.top = new FormAttachment(wFlagField, margin);
+ fdlKeys.top = new FormAttachment(wDiffField, margin);
wlKeys.setLayoutData(fdlKeys);
- int nrKeyRows = (input.getKeyFields() != null ?
input.getKeyFields().length : 1);
+ int nrKeyRows = (input.getKeyFields() != null ?
input.getKeyFields().size() : 1);
ColumnInfo[] ciKeys =
new ColumnInfo[] {
@@ -260,10 +279,10 @@ public class MergeRowsDialog extends BaseTransformDialog {
PropsUi.setLook(wlValues);
FormData fdlValues = new FormData();
fdlValues.left = new FormAttachment(50, 0);
- fdlValues.top = new FormAttachment(wFlagField, margin);
+ fdlValues.top = new FormAttachment(wDiffField, margin);
wlValues.setLayoutData(fdlValues);
- int nrValueRows = (input.getValueFields() != null ?
input.getValueFields().length : 1);
+ int nrValueRows = (input.getValueFields() != null ?
input.getValueFields().size() : 1);
ColumnInfo[] ciValues =
new ColumnInfo[] {
@@ -304,21 +323,16 @@ public class MergeRowsDialog extends BaseTransformDialog {
wReference.setText(Const.NVL(infoStreams.get(0).getTransformName(), ""));
wCompare.setText(Const.NVL(infoStreams.get(1).getTransformName(), ""));
- if (input.getFlagField() != null) {
- wFlagField.setText(input.getFlagField());
- }
+ wFlagField.setText(Const.NVL(input.getFlagField(), ""));
+ wDiffField.setText(Const.NVL(input.getDiffJsonField(), ""));
- for (int i = 0; i < input.getKeyFields().length; i++) {
+ for (int i = 0; i < input.getKeyFields().size(); i++) {
TableItem item = wKeys.table.getItem(i);
- if (input.getKeyFields()[i] != null) {
- item.setText(1, input.getKeyFields()[i]);
- }
+ item.setText(1, Const.NVL(input.getKeyFields().get(i), ""));
}
- for (int i = 0; i < input.getValueFields().length; i++) {
+ for (int i = 0; i < input.getValueFields().size(); i++) {
TableItem item = wValues.table.getItem(i);
- if (input.getValueFields()[i] != null) {
- item.setText(1, input.getValueFields()[i]);
- }
+ item.setText(1, Const.NVL(input.getValueFields().get(i), ""));
}
wTransformName.selectAll();
@@ -336,24 +350,27 @@ public class MergeRowsDialog extends BaseTransformDialog {
return;
}
+ input.setReferenceTransform(wReference.getText());
+ input.setCompareTransform(wCompare.getText());
List<IStream> infoStreams = input.getTransformIOMeta().getInfoStreams();
infoStreams.get(0).setTransformMeta(pipelineMeta.findTransform(wReference.getText()));
infoStreams.get(1).setTransformMeta(pipelineMeta.findTransform(wCompare.getText()));
input.setFlagField(wFlagField.getText());
+ input.setDiffJsonField(wDiffField.getText());
int nrKeys = wKeys.nrNonEmpty();
int nrValues = wValues.nrNonEmpty();
- input.allocate(nrKeys, nrValues);
-
+ input.getKeyFields().clear();
for (int i = 0; i < nrKeys; i++) {
TableItem item = wKeys.getNonEmpty(i);
- input.getKeyFields()[i] = item.getText(1);
+ input.getKeyFields().add(item.getText(1));
}
+ input.getValueFields().clear();
for (int i = 0; i < nrValues; i++) {
TableItem item = wValues.getNonEmpty(i);
- input.getValueFields()[i] = item.getText(1);
+ input.getValueFields().add(item.getText(1));
}
transformName = wTransformName.getText(); // return value
diff --git
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMeta.java
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMeta.java
index d59bbb7962..c784972431 100644
---
a/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMeta.java
+++
b/plugins/transforms/mergerows/src/main/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMeta.java
@@ -17,23 +17,23 @@
package org.apache.hop.pipeline.transforms.mergerows;
+import java.util.ArrayList;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang.StringUtils;
import org.apache.hop.core.CheckResult;
-import org.apache.hop.core.Const;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopRowException;
import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.exception.HopXmlException;
-import org.apache.hop.core.injection.Injection;
-import org.apache.hop.core.injection.InjectionSupported;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
-import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.PipelineMeta.PipelineType;
@@ -45,9 +45,9 @@ import org.apache.hop.pipeline.transform.stream.IStream;
import org.apache.hop.pipeline.transform.stream.IStream.StreamType;
import org.apache.hop.pipeline.transform.stream.Stream;
import org.apache.hop.pipeline.transform.stream.StreamIcon;
-import org.w3c.dom.Node;
-@InjectionSupported(localizationPrefix = "MergeRows.Injection.")
+@Getter
+@Setter
@Transform(
id = "MergeRows",
image = "mergerows.svg",
@@ -58,152 +58,71 @@ import org.w3c.dom.Node;
documentationUrl = "/pipeline/transforms/mergerows.html")
public class MergeRowsMeta extends BaseTransformMeta<MergeRows, MergeRowsData>
{
private static final Class<?> PKG = MergeRowsMeta.class;
- public static final String CONST_VALUE = "value";
- @Injection(name = "FLAG_FIELD")
+ @HopMetadataProperty(
+ key = "flag_field",
+ injectionKey = "FLAG_FIELD",
+ injectionKeyDescription = "MergeRows.Injection.FLAG_FIELD")
private String flagField;
- @Injection(name = "KEY_FIELDS")
- private String[] keyFields;
-
- @Injection(name = "VALUE_FIELDS")
- private String[] valueFields;
-
- /**
- * @return Returns the keyFields.
- */
- public String[] getKeyFields() {
- return keyFields;
- }
-
- /**
- * @param keyFields The keyFields to set.
- */
- public void setKeyFields(String[] keyFields) {
- this.keyFields = keyFields;
- }
-
- /**
- * @return Returns the valueFields.
- */
- public String[] getValueFields() {
- return valueFields;
- }
-
- /**
- * @param valueFields The valueFields to set.
- */
- public void setValueFields(String[] valueFields) {
- this.valueFields = valueFields;
- }
+ @HopMetadataProperty(
+ key = "key",
+ groupKey = "keys",
+ injectionKey = "KEY_FIELD",
+ injectionKeyDescription = "MergeRows.Injection.KEY_FIELD",
+ injectionGroupKey = "KEY_FIELDS",
+ injectionGroupDescription = "MergeRows.Injection.KEY_FIELDS")
+ private List<String> keyFields;
+
+ @HopMetadataProperty(
+ key = "value",
+ groupKey = "values",
+ injectionKey = "VALUE_FIELD",
+ injectionKeyDescription = "MergeRows.Injection.VALUE_FIELD",
+ injectionGroupKey = "VALUE_FIELDS",
+ injectionGroupDescription = "MergeRows.Injection.VALUE_FIELDS")
+ private List<String> valueFields;
+
+ @HopMetadataProperty(
+ key = "reference",
+ injectionKey = "REFERENCE_TRANSFORM",
+ injectionKeyDescription =
"MergeRowsMeta.InfoStream.FirstStream.Description")
+ private String referenceTransform;
+
+ @HopMetadataProperty(
+ key = "compare",
+ injectionKey = "COMPARE_TRANSFORM",
+ injectionKeyDescription =
"MergeRowsMeta.InfoStream.SecondStream.Description")
+ private String compareTransform;
+
+ @HopMetadataProperty(
+ key = "diff-field",
+ injectionKey = "DIFF_FIELD",
+ injectionKeyDescription = "MergeRows.Injection.DIFF_FIELD")
+ private String diffJsonField;
public MergeRowsMeta() {
- super(); // allocate BaseTransformMeta
- }
-
- @Override
- public void loadXml(Node transformNode, IHopMetadataProvider
metadataProvider)
- throws HopXmlException {
- readData(transformNode);
- }
-
- /**
- * @return Returns the flagField.
- */
- public String getFlagField() {
- return flagField;
- }
-
- /**
- * @param flagField The flagField to set.
- */
- public void setFlagField(String flagField) {
- this.flagField = flagField;
+ super();
+ keyFields = new ArrayList<>();
+ valueFields = new ArrayList<>();
}
- public void allocate(int nrKeys, int nrValues) {
- keyFields = new String[nrKeys];
- valueFields = new String[nrValues];
+ public MergeRowsMeta(MergeRowsMeta m) {
+ this.flagField = m.flagField;
+ this.keyFields = new ArrayList<>(m.keyFields);
+ this.valueFields = new ArrayList<>(m.valueFields);
+ this.referenceTransform = m.referenceTransform;
+ this.compareTransform = m.compareTransform;
}
@Override
- public Object clone() {
- MergeRowsMeta retval = (MergeRowsMeta) super.clone();
- int nrKeys = keyFields.length;
- int nrValues = valueFields.length;
- retval.allocate(nrKeys, nrValues);
- System.arraycopy(keyFields, 0, retval.keyFields, 0, nrKeys);
- System.arraycopy(valueFields, 0, retval.valueFields, 0, nrValues);
- return retval;
- }
-
- @Override
- public String getXml() {
- StringBuilder retval = new StringBuilder();
-
- retval.append(" <keys>" + Const.CR);
- for (int i = 0; i < keyFields.length; i++) {
- retval.append(" " + XmlHandler.addTagValue("key", keyFields[i]));
- }
- retval.append(" </keys>" + Const.CR);
-
- retval.append(" <values>" + Const.CR);
- for (int i = 0; i < valueFields.length; i++) {
- retval.append(" " + XmlHandler.addTagValue(CONST_VALUE,
valueFields[i]));
- }
- retval.append(" </values>" + Const.CR);
-
- retval.append(XmlHandler.addTagValue("flag_field", flagField));
-
- List<IStream> infoStreams = getTransformIOMeta().getInfoStreams();
- retval.append(XmlHandler.addTagValue("reference",
infoStreams.get(0).getTransformName()));
- retval.append(XmlHandler.addTagValue("compare",
infoStreams.get(1).getTransformName()));
- retval.append(" <compare>" + Const.CR);
-
- retval.append(" </compare>" + Const.CR);
-
- return retval.toString();
- }
-
- private void readData(Node transformNode) throws HopXmlException {
- try {
-
- Node keysnode = XmlHandler.getSubNode(transformNode, "keys");
- Node valuesnode = XmlHandler.getSubNode(transformNode, "values");
-
- int nrKeys = XmlHandler.countNodes(keysnode, "key");
- int nrValues = XmlHandler.countNodes(valuesnode, CONST_VALUE);
-
- allocate(nrKeys, nrValues);
-
- for (int i = 0; i < nrKeys; i++) {
- Node keynode = XmlHandler.getSubNodeByNr(keysnode, "key", i);
- keyFields[i] = XmlHandler.getNodeValue(keynode);
- }
-
- for (int i = 0; i < nrValues; i++) {
- Node valuenode = XmlHandler.getSubNodeByNr(valuesnode, CONST_VALUE, i);
- valueFields[i] = XmlHandler.getNodeValue(valuenode);
- }
-
- flagField = XmlHandler.getTagValue(transformNode, "flag_field");
-
- List<IStream> infoStreams = getTransformIOMeta().getInfoStreams();
- IStream referenceStream = infoStreams.get(0);
- IStream compareStream = infoStreams.get(1);
-
- compareStream.setSubject(XmlHandler.getTagValue(transformNode,
"compare"));
- referenceStream.setSubject(XmlHandler.getTagValue(transformNode,
"reference"));
- } catch (Exception e) {
- throw new HopXmlException(
- BaseMessages.getString(PKG,
"MergeRowsMeta.Exception.UnableToLoadTransformMeta"), e);
- }
+ public MergeRowsMeta clone() {
+ return new MergeRowsMeta(this);
}
@Override
public void setDefault() {
flagField = "flagfield";
- allocate(0, 0);
}
@Override
@@ -214,14 +133,6 @@ public class MergeRowsMeta extends
BaseTransformMeta<MergeRows, MergeRowsData> {
}
}
- public boolean chosesTargetTransforms() {
- return false;
- }
-
- public String[] getTargetTransforms() {
- return null;
- }
-
@Override
public void getFields(
IRowMeta r,
@@ -251,6 +162,12 @@ public class MergeRowsMeta extends
BaseTransformMeta<MergeRows, MergeRowsData> {
IValueMeta flagFieldValue = new ValueMetaString(flagField);
flagFieldValue.setOrigin(name);
r.addValueMeta(flagFieldValue);
+
+ if (StringUtils.isNotEmpty(variables.resolve(diffJsonField))) {
+ IValueMeta diffField = new
ValueMetaString(variables.resolve(diffJsonField));
+ diffField.setOrigin(name);
+ r.addValueMeta(diffField);
+ }
}
@Override
@@ -313,7 +230,7 @@ public class MergeRowsMeta extends
BaseTransformMeta<MergeRows, MergeRowsData> {
MergeRows.checkInputLayoutValid(referenceRowMeta, compareRowMeta);
rowsMatch = true;
} catch (HopRowException kre) {
- rowsMatch = false;
+ // Ignore
}
if (rowsMatch) {
cr =
@@ -338,7 +255,6 @@ public class MergeRowsMeta extends
BaseTransformMeta<MergeRows, MergeRowsData> {
public ITransformIOMeta getTransformIOMeta() {
ITransformIOMeta ioMeta = super.getTransformIOMeta(false);
if (ioMeta == null) {
-
ioMeta = new TransformIOMeta(true, true, false, false, false, false);
ioMeta.addStream(
@@ -347,25 +263,20 @@ public class MergeRowsMeta extends
BaseTransformMeta<MergeRows, MergeRowsData> {
null,
BaseMessages.getString(PKG,
"MergeRowsMeta.InfoStream.FirstStream.Description"),
StreamIcon.INFO,
- null));
+ referenceTransform));
ioMeta.addStream(
new Stream(
StreamType.INFO,
null,
BaseMessages.getString(PKG,
"MergeRowsMeta.InfoStream.SecondStream.Description"),
StreamIcon.INFO,
- null));
+ compareTransform));
setTransformIOMeta(ioMeta);
}
return ioMeta;
}
- @Override
- public void resetTransformIoMeta() {
- // Do Nothing
- }
-
@Override
public PipelineType[] getSupportedPipelineTypes() {
return new PipelineType[] {
diff --git
a/plugins/transforms/mergerows/src/main/resources/org/apache/hop/pipeline/transforms/mergerows/messages/messages_en_US.properties
b/plugins/transforms/mergerows/src/main/resources/org/apache/hop/pipeline/transforms/mergerows/messages/messages_en_US.properties
index ee4b0ac191..566d8c9e3f 100644
---
a/plugins/transforms/mergerows/src/main/resources/org/apache/hop/pipeline/transforms/mergerows/messages/messages_en_US.properties
+++
b/plugins/transforms/mergerows/src/main/resources/org/apache/hop/pipeline/transforms/mergerows/messages/messages_en_US.properties
@@ -19,8 +19,11 @@ MergeRows.Description=Merge two streams of rows, sorted on a
certain key. The tw
MergeRows.Exception.InvalidLayoutDetected=Invalid layout detected in input
streams, keys and values to merge have to be of identical structure and be in
the same place in the rows
MergeRows.Exception.UnableToFindFieldInReferenceStream=Unable to find field
[{0}] in reference stream.
MergeRows.Injection.FLAG_FIELD=The name of the flag field.
+MergeRows.Injection.DIFF_FIELD=The name of the field for the difference JSON.
MergeRows.Injection.KEY_FIELDS=Specify key fields to compare.
+MergeRows.Injection.KEY_FIELD=The key field to match with
MergeRows.Injection.VALUE_FIELDS=Specify value fields to compare.
+MergeRows.Injection.VALUE_FIELD=The field to compare
MergeRows.LineNumber=linenr
MergeRows.Log.BothTrueAndFalseNeeded=Both the ''true'' and the ''false''
transforms need to be supplied, or neither
MergeRows.Log.DataInfo=ONE\: {0} / TWO\:
@@ -30,7 +33,8 @@ MergeRowsDialog.ColumnInfo.ValueField=Value field
MergeRowsDialog.Compare.Label=Compare rows origin\:
MergeRowsDialog.ErrorGettingFields.DialogMessage=Unable to get the fields
because of an error\:
MergeRowsDialog.ErrorGettingFields.DialogTitle=Error getting fields
-MergeRowsDialog.FlagField.Label=Flag fieldname
+MergeRowsDialog.FlagField.Label=Flag field name
+MergeRowsDialog.DiffField.Label=Difference JSON field name
MergeRowsDialog.KeyFields.Button=Get &key fields
MergeRowsDialog.Keys.Label=Keys to match \:
MergeRowsDialog.MergeRowsWarningDialog.DialogMessage=If the incoming data is
not sorted on the specified keys, the output results may not be correct. We
recommend sorting the incoming data within the pipeline.
diff --git
a/plugins/transforms/mergerows/src/test/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMetaInjectionTest.java
b/plugins/transforms/mergerows/src/test/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMetaInjectionTest.java
deleted file mode 100644
index 0727e9f0aa..0000000000
---
a/plugins/transforms/mergerows/src/test/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMetaInjectionTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.pipeline.transforms.mergerows;
-
-import org.apache.hop.core.injection.BaseMetadataInjectionTestJunit5;
-import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-public class MergeRowsMetaInjectionTest extends
BaseMetadataInjectionTestJunit5<MergeRowsMeta> {
- @RegisterExtension
- static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
-
- @BeforeEach
- public void setup() throws Exception {
- setup(new MergeRowsMeta());
- }
-
- @Test
- public void test() throws Exception {
- check("FLAG_FIELD", () -> meta.getFlagField());
- check("KEY_FIELDS", () -> meta.getKeyFields()[0]);
- check("VALUE_FIELDS", () -> meta.getValueFields()[0]);
- }
-}
diff --git
a/plugins/transforms/mergerows/src/test/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMetaTest.java
b/plugins/transforms/mergerows/src/test/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMetaTest.java
index b957584263..3083170635 100644
---
a/plugins/transforms/mergerows/src/test/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMetaTest.java
+++
b/plugins/transforms/mergerows/src/test/java/org/apache/hop/pipeline/transforms/mergerows/MergeRowsMetaTest.java
@@ -6,80 +6,51 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
+ *
*/
package org.apache.hop.pipeline.transforms.mergerows;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
import org.apache.hop.core.HopEnvironment;
-import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.plugins.PluginRegistry;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
-import org.apache.hop.pipeline.transform.ITransformMeta;
-import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
-import org.apache.hop.pipeline.transforms.loadsave.initializer.IInitializer;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.ArrayLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.StringLoadSaveValidator;
+import org.apache.hop.pipeline.transform.TransformSerializationTestUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
-public class MergeRowsMetaTest implements IInitializer<ITransformMeta> {
- LoadSaveTester loadSaveTester;
- Class<MergeRowsMeta> testMetaClass = MergeRowsMeta.class;
+class MergeRowsMetaTest {
@RegisterExtension
static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
@BeforeEach
- public void setUpLoadSave() throws Exception {
+ void setUpLoadSave() throws Exception {
HopEnvironment.init();
PluginRegistry.init();
- List<String> attributes = Arrays.asList("flagField", "keyFields",
"valueFields");
-
- IFieldLoadSaveValidator<String[]> stringArrayLoadSaveValidator =
- new ArrayLoadSaveValidator<>(new StringLoadSaveValidator(), 5);
-
- Map<String, IFieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<>();
- attrValidatorMap.put("keyFields", stringArrayLoadSaveValidator);
- attrValidatorMap.put("valueFields", stringArrayLoadSaveValidator);
-
- Map<String, IFieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<>();
-
- loadSaveTester =
- new LoadSaveTester(
- testMetaClass,
- attributes,
- new HashMap<>(),
- new HashMap<>(),
- attrValidatorMap,
- typeValidatorMap,
- this);
- }
-
- // Call the allocate method on the LoadSaveTester meta class
- @Override
- public void modify(ITransformMeta someMeta) {
- if (someMeta instanceof MergeRowsMeta) {
- ((MergeRowsMeta) someMeta).allocate(5, 5);
- }
}
@Test
- public void testSerialization() throws HopException {
- loadSaveTester.testSerialization();
+ void testSerialization() throws Exception {
+ MergeRowsMeta meta =
+ TransformSerializationTestUtil.testSerialization("/merge-rows.xml",
MergeRowsMeta.class);
+
+ assertEquals(1, meta.getKeyFields().size());
+ assertEquals("id", meta.getKeyFields().get(0));
+ assertEquals("flagfield", meta.getFlagField());
+ assertEquals("source1", meta.getReferenceTransform());
+ assertEquals("source2", meta.getCompareTransform());
+ assertEquals(2, meta.getValueFields().size());
+ assertEquals("name", meta.getValueFields().get(0));
+ assertEquals("score", meta.getValueFields().get(1));
}
-
- // Note - cloneTest removed as load/save validator covers clone testing as
well now.
}
diff --git a/plugins/transforms/mergerows/src/test/resources/merge-rows.xml
b/plugins/transforms/mergerows/src/test/resources/merge-rows.xml
new file mode 100644
index 0000000000..7984c65343
--- /dev/null
+++ b/plugins/transforms/mergerows/src/test/resources/merge-rows.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<transform>
+ <keys>
+ <key>id</key>
+ </keys>
+ <values>
+ <value>name</value>
+ <value>score</value>
+ </values>
+ <flag_field>flagfield</flag_field>
+ <reference>source1</reference>
+ <compare>source2</compare>
+</transform>
\ No newline at end of file
diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/HopCommandGui.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/HopCommandGui.java
index efac41f7cc..40a11e7b9b 100644
--- a/ui/src/main/java/org/apache/hop/ui/hopgui/HopCommandGui.java
+++ b/ui/src/main/java/org/apache/hop/ui/hopgui/HopCommandGui.java
@@ -20,6 +20,7 @@ package org.apache.hop.ui.hopgui;
import lombok.Getter;
import lombok.Setter;
+import org.apache.hop.core.Const;
import org.apache.hop.core.HopVersionProvider;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.variables.IVariables;
@@ -45,6 +46,7 @@ public class HopCommandGui implements Runnable, IHopCommand {
CommandLine cmd, IVariables variables, MultiMetadataProvider
metadataProvider)
throws HopException {
// Nothing specific
+ System.setProperty(Const.HOP_PLATFORM_RUNTIME, "GUI");
}
@Override
