This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 9a9bf8bfd2 issue #1946 (Unique Rows by Hash set XML cleanup, MDI)
(#6794)
9a9bf8bfd2 is described below
commit 9a9bf8bfd2921e4639956da81ba82a0d64008f40
Author: Matt Casters <[email protected]>
AuthorDate: Tue Mar 17 09:27:21 2026 +0100
issue #1946 (Unique Rows by Hash set XML cleanup, MDI) (#6794)
---
.../mdi/0040-unique-hashset-template.hpl | 66 +++++
.../mdi/0040-unique-hashset-validate.hpl | 255 +++++++++++++++++
integration-tests/mdi/0040-unique-hashset.hpl | 278 ++++++++++++++++++
.../datasets/golden-unique-rows-hashset-fields.csv | 4 +
.../mdi/datasets/golden-unique-rows-hashset.csv | 2 +
.../mdi/files/unique-rows-hashset.xml | 34 +++
.../main-0040-unique-hashset.hwf} | 63 ++++-
.../dataset/golden-unique-rows-hashset-fields.json | 16 ++
.../dataset/golden-unique-rows-hashset.json | 32 +++
.../0040-unique-hashset-validate UNIT.json | 74 +++++
.../transforms/0034-unique-rows-hashset.hpl | 309 +++++++++++++++++++++
.../datasets/golden-unique-rows-hashset.csv | 6 +
.../transforms/main-0034-unique-rows.hwf | 15 +-
.../dataset/golden-unique-rows-hashset.json | 32 +++
.../unit-test/0034-unique-rows-hashset UNIT.json | 34 +++
.../uniquerowsbyhashset/UniqueRowsByHashSet.java | 15 +-
.../UniqueRowsByHashSetDialog.java | 19 +-
.../UniqueRowsByHashSetMeta.java | 203 ++++----------
.../messages/messages_en_US.properties | 6 +
.../UniqueRowsByHashSetMetaTest.java | 89 +++---
.../src/test/resources/transform.xml | 33 +++
21 files changed, 1356 insertions(+), 229 deletions(-)
diff --git a/integration-tests/mdi/0040-unique-hashset-template.hpl
b/integration-tests/mdi/0040-unique-hashset-template.hpl
new file mode 100644
index 0000000000..d7db03cdd9
--- /dev/null
+++ b/integration-tests/mdi/0040-unique-hashset-template.hpl
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0040-unique-hashset-template</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/03/16 15:28:54.628</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/16 15:28:54.628</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ </order>
+ <transform>
+ <name>Unique rows (HashSet)</name>
+ <type>UniqueRowsByHashSet</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+</fields>
+ <reject_duplicate_row>N</reject_duplicate_row>
+ <store_values>Y</store_values>
+ <attributes/>
+ <GUI>
+ <xloc>176</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/mdi/0040-unique-hashset-validate.hpl
b/integration-tests/mdi/0040-unique-hashset-validate.hpl
new file mode 100644
index 0000000000..89cdc07028
--- /dev/null
+++ b/integration-tests/mdi/0040-unique-hashset-validate.hpl
@@ -0,0 +1,255 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0040-unique-hashset-validate</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/03/16 15:40:51.338</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/16 15:40:51.338</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>${java.io.tmpdir}/unique-rows-hashset.hpl</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>${java.io.tmpdir}/unique-rows-hashset.hpl fields</from>
+ <to>validate fields</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>${java.io.tmpdir}/unique-rows-hashset.hpl</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <XmlField/>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <extensionFieldName/>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>error_description</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>error_description</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>reject_duplicate_row</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>reject_duplicate_row</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>store_values</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>store_values</xPath>
+ </field>
+ </fields>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/unique-rows-hashset.hpl</name>
+ </file>
+ <hiddenFieldName/>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
+ <limit>0</limit>
+ <loopxpath>/pipeline/transform</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
+ <readurl>N</readurl>
+ <rootUriNameFieldName/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>${java.io.tmpdir}/unique-rows-hashset.hpl fields</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <XmlField/>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <extensionFieldName/>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>name</xPath>
+ </field>
+ </fields>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/unique-rows-hashset.hpl</name>
+ </file>
+ <hiddenFieldName/>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
+ <limit>0</limit>
+ <loopxpath>/pipeline/transform/fields/field</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
+ <readurl>N</readurl>
+ <rootUriNameFieldName/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate fields</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/mdi/0040-unique-hashset.hpl
b/integration-tests/mdi/0040-unique-hashset.hpl
new file mode 100644
index 0000000000..f30e99909d
--- /dev/null
+++ b/integration-tests/mdi/0040-unique-hashset.hpl
@@ -0,0 +1,278 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0040-unique-hashset</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/03/16 15:29:25.284</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/16 15:29:25.284</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>unique-rows-hashset.xml</from>
+ <to>ETL metadata injection</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>unique-rows-hashset.xml fields</from>
+ <to>ETL metadata injection</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>unique-rows-hashset.xml</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <XmlField/>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <extensionFieldName/>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>store_values</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>store_values</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>reject_duplicate_row</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>reject_duplicate_row</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>error_description</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>error_description</xPath>
+ </field>
+ </fields>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${PROJECT_HOME}/files/unique-rows-hashset.xml</name>
+ </file>
+ <hiddenFieldName/>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
+ <limit>0</limit>
+ <loopxpath>/transform</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
+ <readurl>N</readurl>
+ <rootUriNameFieldName/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>unique-rows-hashset.xml fields</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <XmlField/>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <extensionFieldName/>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>name</xPath>
+ </field>
+ </fields>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${PROJECT_HOME}/files/unique-rows-hashset.xml</name>
+ </file>
+ <hiddenFieldName/>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
+ <limit>0</limit>
+ <loopxpath>/transform/fields/field</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
+ <readurl>N</readurl>
+ <rootUriNameFieldName/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>176</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>ETL metadata injection</name>
+ <type>MetaInject</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <filename>${PROJECT_HOME}/0040-unique-hashset-template.hpl</filename>
+ <run_configuration>local</run_configuration>
+ <source_transform/>
+ <source_output_fields> </source_output_fields>
+ <target_file>${java.io.tmpdir}/unique-rows-hashset.hpl</target_file>
+ <create_parent_folder>Y</create_parent_folder>
+ <no_execution>Y</no_execution>
+ <allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
+ <stream_source_transform/>
+ <stream_target_transform/>
+ <mappings>
+ <mapping>
+ <target_transform_name>Unique rows (HashSet)</target_transform_name>
+ <target_attribute_key>COMPARE_FIELD_NAME</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>unique-rows-hashset.xml fields</source_transform>
+ <source_field>name</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>Unique rows (HashSet)</target_transform_name>
+ <target_attribute_key>ERROR_DESCRIPTION</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>unique-rows-hashset.xml</source_transform>
+ <source_field>error_description</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>Unique rows (HashSet)</target_transform_name>
+ <target_attribute_key>REJECT_DUPLICATE_ROW</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>unique-rows-hashset.xml</source_transform>
+ <source_field>reject_duplicate_row</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>Unique rows (HashSet)</target_transform_name>
+ <target_attribute_key>STORE_VALUES</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>unique-rows-hashset.xml</source_transform>
+ <source_field>store_values</source_field>
+ </mapping>
+ </mappings>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/mdi/datasets/golden-unique-rows-hashset-fields.csv
b/integration-tests/mdi/datasets/golden-unique-rows-hashset-fields.csv
new file mode 100644
index 0000000000..68cc89633c
--- /dev/null
+++ b/integration-tests/mdi/datasets/golden-unique-rows-hashset-fields.csv
@@ -0,0 +1,4 @@
+name
+compareField1
+compareField2
+compareField3
diff --git a/integration-tests/mdi/datasets/golden-unique-rows-hashset.csv
b/integration-tests/mdi/datasets/golden-unique-rows-hashset.csv
new file mode 100644
index 0000000000..e57944bc4d
--- /dev/null
+++ b/integration-tests/mdi/datasets/golden-unique-rows-hashset.csv
@@ -0,0 +1,2 @@
+store_values,reject_duplicate_row,error_description
+Y,Y,Duplicate row detected
diff --git a/integration-tests/mdi/files/unique-rows-hashset.xml
b/integration-tests/mdi/files/unique-rows-hashset.xml
new file mode 100644
index 0000000000..048b921f11
--- /dev/null
+++ b/integration-tests/mdi/files/unique-rows-hashset.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<transform>
+ <store_values>Y</store_values>
+ <reject_duplicate_row>Y</reject_duplicate_row>
+ <error_description>Duplicate row detected</error_description>
+ <fields>
+ <field>
+ <name>compareField1</name>
+ </field>
+ <field>
+ <name>compareField2</name>
+ </field>
+ <field>
+ <name>compareField3</name>
+ </field>
+ </fields>
+</transform>
+
diff --git a/integration-tests/transforms/main-0034-unique-rows.hwf
b/integration-tests/mdi/main-0040-unique-hashset.hwf
similarity index 56%
copy from integration-tests/transforms/main-0034-unique-rows.hwf
copy to integration-tests/mdi/main-0040-unique-hashset.hwf
index 083945f40a..7b7648162f 100644
--- a/integration-tests/transforms/main-0034-unique-rows.hwf
+++ b/integration-tests/mdi/main-0040-unique-hashset.hwf
@@ -18,16 +18,15 @@ limitations under the License.
-->
<workflow>
- <name>main-0034-unique-rows</name>
+ <name>main-0040-unique-hashset</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<workflow_version/>
- <workflow_status>0</workflow_status>
<created_user>-</created_user>
- <created_date>2021/07/21 20:36:24.295</created_date>
+ <created_date>2026/03/16 15:28:51.665</created_date>
<modified_user>-</modified_user>
- <modified_date>2021/07/21 20:36:24.295</modified_date>
+ <modified_date>2026/03/16 15:28:51.665</modified_date>
<parameters>
</parameters>
<actions>
@@ -36,31 +35,58 @@ limitations under the License.
<description/>
<type>SPECIAL</type>
<attributes/>
- <repeat>N</repeat>
- <schedulerType>0</schedulerType>
- <intervalSeconds>0</intervalSeconds>
- <intervalMinutes>60</intervalMinutes>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
<hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
<minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
<weekDay>1</weekDay>
- <DayOfMonth>1</DayOfMonth>
<parallel>N</parallel>
- <xloc>128</xloc>
+ <xloc>96</xloc>
<yloc>80</yloc>
<attributes_hac/>
</action>
<action>
- <name>Run Clone Row Tests</name>
+ <name>Run Pipeline Unit Tests</name>
<description/>
<type>RunPipelineTests</type>
<attributes/>
<test_names>
<test_name>
- <name>0034-unique-rows UNIT</name>
+ <name>0040-unique-hashset-validate UNIT</name>
</test_name>
</test_names>
<parallel>N</parallel>
- <xloc>272</xloc>
+ <xloc>416</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0040-unique-hashset</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0040-unique-hashset.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
<yloc>80</yloc>
<attributes_hac/>
</action>
@@ -68,11 +94,18 @@ limitations under the License.
<hops>
<hop>
<from>Start</from>
- <to>Run Clone Row Tests</to>
+ <to>0040-unique-hashset</to>
<enabled>Y</enabled>
- <evaluation>N</evaluation>
+ <evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
+ <hop>
+ <from>0040-unique-hashset</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
</hops>
<notepads>
</notepads>
diff --git
a/integration-tests/mdi/metadata/dataset/golden-unique-rows-hashset-fields.json
b/integration-tests/mdi/metadata/dataset/golden-unique-rows-hashset-fields.json
new file mode 100644
index 0000000000..ab29b163a2
--- /dev/null
+++
b/integration-tests/mdi/metadata/dataset/golden-unique-rows-hashset-fields.json
@@ -0,0 +1,16 @@
+{
+ "base_filename": "golden-unique-rows-hashset-fields.csv",
+ "name": "golden-unique-rows-hashset-fields",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/mdi/metadata/dataset/golden-unique-rows-hashset.json
b/integration-tests/mdi/metadata/dataset/golden-unique-rows-hashset.json
new file mode 100644
index 0000000000..a8a650d326
--- /dev/null
+++ b/integration-tests/mdi/metadata/dataset/golden-unique-rows-hashset.json
@@ -0,0 +1,32 @@
+{
+ "base_filename": "golden-unique-rows-hashset.csv",
+ "name": "golden-unique-rows-hashset",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "store_values",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "reject_duplicate_row",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "error_description",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/mdi/metadata/unit-test/0040-unique-hashset-validate
UNIT.json
b/integration-tests/mdi/metadata/unit-test/0040-unique-hashset-validate
UNIT.json
new file mode 100644
index 0000000000..af8ecd66a4
--- /dev/null
+++ b/integration-tests/mdi/metadata/unit-test/0040-unique-hashset-validate
UNIT.json
@@ -0,0 +1,74 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "error_description",
+ "data_set_field": "error_description"
+ },
+ {
+ "transform_field": "reject_duplicate_row",
+ "data_set_field": "reject_duplicate_row"
+ },
+ {
+ "transform_field": "store_values",
+ "data_set_field": "store_values"
+ }
+ ],
+ "field_order": [
+ "store_values",
+ "reject_duplicate_row",
+ "error_description"
+ ],
+ "data_set_name": "golden-unique-rows-hashset",
+ "transform_name": "Dummy (do nothing)"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "error_description",
+ "data_set_field": "error_description"
+ },
+ {
+ "transform_field": "reject_duplicate_row",
+ "data_set_field": "reject_duplicate_row"
+ },
+ {
+ "transform_field": "store_values",
+ "data_set_field": "store_values"
+ }
+ ],
+ "field_order": [
+ "store_values",
+ "reject_duplicate_row",
+ "error_description"
+ ],
+ "data_set_name": "golden-unique-rows-hashset",
+ "transform_name": "validate"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ }
+ ],
+ "field_order": [
+ "name"
+ ],
+ "data_set_name": "golden-unique-rows-hashset-fields",
+ "transform_name": "validate fields"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0040-unique-hashset-validate UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0040-unique-hashset-validate.hpl"
+}
\ No newline at end of file
diff --git a/integration-tests/transforms/0034-unique-rows-hashset.hpl
b/integration-tests/transforms/0034-unique-rows-hashset.hpl
new file mode 100644
index 0000000000..0386abc432
--- /dev/null
+++ b/integration-tests/transforms/0034-unique-rows-hashset.hpl
@@ -0,0 +1,309 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0034-unique-rows-hashset</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/07/13 21:09:00.016</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/07/13 21:09:00.016</modified_date>
+ </info>
+ <notepads>
+ <notepad>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorred>201</backgroundcolorred>
+ <bordercolorblue>90</bordercolorblue>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorred>14</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>90</fontcolorblue>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorred>14</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontname>Segoe UI</fontname>
+ <fontsize>9</fontsize>
+ <height>-1</height>
+ <xloc>112</xloc>
+ <yloc>32</yloc>
+ <note>Warning, row are sorted in case insentive order and dataset is
case sensitive order</note>
+ <width>437</width>
+ </notepad>
+ </notepads>
+ <order>
+ <hop>
+ <from>Data grid</from>
+ <to>sort</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>sort</from>
+ <to>Unique rows (HashSet)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Unique rows (HashSet)</from>
+ <to>Result</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Unique rows (HashSet)</from>
+ <to>Duplicate row</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Data grid</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>ABCD</item>
+ <item>Z</item>
+ <item>1</item>
+ </line>
+ <line>
+ <item>ABCD</item>
+ <item>Z</item>
+ <item>2</item>
+ </line>
+ <line>
+ <item>ABCD</item>
+ <item>z</item>
+ <item>2</item>
+ </line>
+ <line>
+ <item>ABCD</item>
+ <item>z</item>
+ <item>6</item>
+ </line>
+ <line>
+ <item>abcd</item>
+ <item>X</item>
+ <item>9</item>
+ </line>
+ <line>
+ <item>abcd</item>
+ <item>X</item>
+ <item>9</item>
+ </line>
+ <line>
+ <item>EFG</item>
+ <item>X</item>
+ <item>8</item>
+ </line>
+ <line>
+ <item>EFG</item>
+ <item>Z</item>
+ <item>1</item>
+ </line>
+ <line>
+ <item>ABCD</item>
+ <item>Z</item>
+ <item>2</item>
+ </line>
+ <line>
+ <item>ABCD</item>
+ <item>z</item>
+ <item>2</item>
+ </line>
+ <line>
+ <item>EFG</item>
+ <item>X</item>
+ <item>6</item>
+ </line>
+ <line>
+ <item>EFG</item>
+ <item>Z</item>
+ <item>8</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>Field_A</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>Field_B</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>Field_C</name>
+ <type>Integer</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Duplicate row</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>240</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Result</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>496</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Unique rows (HashSet)</name>
+ <type>UniqueRowsByHashSet</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <error_description>duplicate</error_description>
+ <fields>
+ <field>
+ <name>Field_A</name>
+ </field>
+ <field>
+ <name>Field_B</name>
+ </field>
+ </fields>
+ <reject_duplicate_row>Y</reject_duplicate_row>
+ <store_values>Y</store_values>
+ <attributes/>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>sort</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <compress>N</compress>
+ <directory>${java.io.tmpdir}</directory>
+ <fields>
+ <field>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <name>Field_A</name>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <name>Field_B</name>
+ <presorted>N</presorted>
+ </field>
+ <field>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <name>Field_C</name>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <sort_size>1000000</sort_size>
+ <unique_rows>N</unique_rows>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ <error>
+ <source_transform>Unique rows (HashSet)</source_transform>
+ <target_transform>Duplicate row</target_transform>
+ <is_enabled>Y</is_enabled>
+ <nr_valuename/>
+ <descriptions_valuename/>
+ <fields_valuename/>
+ <codes_valuename/>
+ <max_errors/>
+ <max_pct_errors/>
+ <min_pct_rows/>
+ </error>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-unique-rows-hashset.csv
b/integration-tests/transforms/datasets/golden-unique-rows-hashset.csv
new file mode 100644
index 0000000000..2b53c4edd5
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-unique-rows-hashset.csv
@@ -0,0 +1,6 @@
+Field_A,Field_B,Field_C
+abcd,X,9
+ABCD,Z,1
+ABCD,z,2
+EFG,X,6
+EFG,Z,1
diff --git a/integration-tests/transforms/main-0034-unique-rows.hwf
b/integration-tests/transforms/main-0034-unique-rows.hwf
index 083945f40a..77ec103def 100644
--- a/integration-tests/transforms/main-0034-unique-rows.hwf
+++ b/integration-tests/transforms/main-0034-unique-rows.hwf
@@ -36,21 +36,22 @@ limitations under the License.
<description/>
<type>SPECIAL</type>
<attributes/>
- <repeat>N</repeat>
- <schedulerType>0</schedulerType>
- <intervalSeconds>0</intervalSeconds>
- <intervalMinutes>60</intervalMinutes>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
<hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
<minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
<weekDay>1</weekDay>
- <DayOfMonth>1</DayOfMonth>
<parallel>N</parallel>
<xloc>128</xloc>
<yloc>80</yloc>
<attributes_hac/>
</action>
<action>
- <name>Run Clone Row Tests</name>
+ <name>Unique Rows Tests</name>
<description/>
<type>RunPipelineTests</type>
<attributes/>
@@ -68,7 +69,7 @@ limitations under the License.
<hops>
<hop>
<from>Start</from>
- <to>Run Clone Row Tests</to>
+ <to>Unique Rows Tests</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-unique-rows-hashset.json
b/integration-tests/transforms/metadata/dataset/golden-unique-rows-hashset.json
new file mode 100644
index 0000000000..7b3447880f
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-unique-rows-hashset.json
@@ -0,0 +1,32 @@
+{
+ "base_filename": "golden-unique-rows-hashset.csv",
+ "name": "golden-unique-rows-hashset",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "Field_A",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "Field_B",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "Field_C",
+ "field_format": "####0;-####0"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0034-unique-rows-hashset
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0034-unique-rows-hashset
UNIT.json
new file mode 100644
index 0000000000..87e01b9de6
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0034-unique-rows-hashset
UNIT.json
@@ -0,0 +1,34 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "Field_A",
+ "data_set_field": "Field_A"
+ },
+ {
+ "transform_field": "Field_B",
+ "data_set_field": "Field_B"
+ },
+ {
+ "transform_field": "Field_C",
+ "data_set_field": "Field_C"
+ }
+ ],
+ "field_order": [],
+ "data_set_name": "golden-unique-rows-hashset",
+ "transform_name": "Result"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0034-unique-rows-hashset UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0034-unique-rows-hashset.hpl"
+}
\ No newline at end of file
diff --git
a/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSet.java
b/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSet.java
index 6b6492784b..cd7424dceb 100644
---
a/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSet.java
+++
b/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSet.java
@@ -61,19 +61,20 @@ public class UniqueRowsByHashSet
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
- data.storeValues = meta.getStoreValues();
+ data.storeValues = meta.isStoreValues();
// ICache lookup of fields
- data.fieldnrs = new int[meta.getCompareFields().length];
+ data.fieldnrs = new int[meta.getCompareFields().size()];
- for (int i = 0; i < meta.getCompareFields().length; i++) {
- data.fieldnrs[i] =
getInputRowMeta().indexOfValue(meta.getCompareFields()[i]);
+ for (int i = 0; i < meta.getCompareFields().size(); i++) {
+ UniqueRowsByHashSetMeta.CompareField compareField =
meta.getCompareFields().get(i);
+ data.fieldnrs[i] =
getInputRowMeta().indexOfValue(compareField.getName());
if (data.fieldnrs[i] < 0) {
logError(
BaseMessages.getString(
PKG,
"UniqueRowsByHashSet.Log.CouldNotFindFieldInRow",
- meta.getCompareFields()[i]));
+ meta.getCompareFields().get(i)));
setErrors(1);
stopAll();
return false;
@@ -81,8 +82,8 @@ public class UniqueRowsByHashSet
if (data.sendDuplicateRows) {
data.compareFields =
data.compareFields == null
- ? meta.getCompareFields()[i]
- : data.compareFields + "," + meta.getCompareFields()[i];
+ ? meta.getCompareFields().get(i).getName()
+ : data.compareFields + "," + meta.getCompareFields().get(i);
}
}
if (data.sendDuplicateRows &&
!Utils.isEmpty(meta.getErrorDescription())) {
diff --git
a/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetDialog.java
b/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetDialog.java
index c060130c15..c9c7b8e1e7 100644
---
a/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetDialog.java
+++
b/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetDialog.java
@@ -27,6 +27,7 @@ import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
+import
org.apache.hop.pipeline.transforms.uniquerowsbyhashset.UniqueRowsByHashSetMeta.CompareField;
import org.apache.hop.ui.core.ConstUi;
import org.apache.hop.ui.core.PropsUi;
import org.apache.hop.ui.core.dialog.BaseDialog;
@@ -184,7 +185,7 @@ public class UniqueRowsByHashSetDialog extends
BaseTransformDialog {
fdlFields.top = new FormAttachment(wSettings, margin);
wlFields.setLayoutData(fdlFields);
- final int FieldsRows = input.getCompareFields() == null ? 0 :
input.getCompareFields().length;
+ final int FieldsRows = input.getCompareFields().size();
colinf =
new ColumnInfo[] {
@@ -257,16 +258,14 @@ public class UniqueRowsByHashSetDialog extends
BaseTransformDialog {
/** Copy information from the meta-data input to the dialog fields. */
public void getData() {
- wStoreValues.setSelection(input.getStoreValues());
+ wStoreValues.setSelection(input.isStoreValues());
wRejectDuplicateRow.setSelection(input.isRejectDuplicateRow());
if (input.getErrorDescription() != null) {
wErrorDesc.setText(input.getErrorDescription());
}
- for (int i = 0; i < input.getCompareFields().length; i++) {
+ for (int i = 0; i < input.getCompareFields().size(); i++) {
TableItem item = wFields.table.getItem(i);
- if (input.getCompareFields()[i] != null) {
- item.setText(1, input.getCompareFields()[i]);
- }
+ item.setText(1, Const.NVL(input.getCompareFields().get(i).getName(),
""));
}
wFields.setRowNums();
wFields.optWidth(true);
@@ -284,12 +283,10 @@ public class UniqueRowsByHashSetDialog extends
BaseTransformDialog {
}
int nrFields = wFields.nrNonEmpty();
- input.allocate(nrFields);
-
- for (int i = 0; i < nrFields; i++) {
- TableItem item = wFields.getNonEmpty(i);
- input.getCompareFields()[i] = item.getText(1);
+ input.getCompareFields().clear();
+ for (TableItem item : wFields.getNonEmptyItems()) {
+ input.getCompareFields().add(new CompareField(item.getText(1)));
}
transformName = wTransformName.getText(); // return value
diff --git
a/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetMeta.java
b/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetMeta.java
index b6d56927ec..4fa440610b 100644
---
a/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetMeta.java
+++
b/plugins/transforms/uniquerowsbyhashset/src/main/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetMeta.java
@@ -17,21 +17,21 @@
package org.apache.hop.pipeline.transforms.uniquerowsbyhashset;
+import java.util.ArrayList;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
-import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.exception.HopXmlException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.variables.IVariables;
-import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.w3c.dom.Node;
@Transform(
id = "UniqueRowsByHashSet",
@@ -41,169 +41,84 @@ import org.w3c.dom.Node;
categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform",
keywords = "i18n::UniqueRowsByHashSetMeta.keyword",
documentationUrl = "/pipeline/transforms/uniquerowsbyhashset.html")
+@Getter
+@Setter
public class UniqueRowsByHashSetMeta
extends BaseTransformMeta<UniqueRowsByHashSet, UniqueRowsByHashSetData> {
private static final Class<?> PKG = UniqueRowsByHashSetMeta.class;
- public static final String CONST_FIELD = "field";
- public static final String CONST_SPACES = " ";
+
+ @Getter
+ @Setter
+ public static class CompareField {
+ @HopMetadataProperty(
+ key = "name",
+ injectionKey = "COMPARE_FIELD_NAME",
+ injectionKeyDescription =
"UniqueRowsByHashSetMeta.Injection.COMPARE_FIELD_NAME")
+ private String name;
+
+ public CompareField() {}
+
+ public CompareField(CompareField f) {
+ this();
+ this.name = f.name;
+ }
+
+ public CompareField(String name) {
+ this.name = name;
+ }
+ }
/**
* Whether to compare strictly by hash value or to store the row values for
strict equality
* checking
*/
+ @HopMetadataProperty(
+ key = "store_values",
+ injectionKey = "STORE_VALUES",
+ injectionKeyDescription =
"UniqueRowsByHashSetMeta.Injection.STORE_VALUES")
private boolean storeValues;
/** The fields to compare for duplicates, null means all */
- private String[] compareFields;
-
+ @HopMetadataProperty(
+ key = "field",
+ groupKey = "fields",
+ injectionKey = "COMPARE_FIELD",
+ injectionGroupKey = "COMPARE_FIELDS",
+ injectionKeyDescription =
"UniqueRowsByHashSetMeta.Injection.COMPARE_FIELD",
+ injectionGroupDescription =
"UniqueRowsByHashSetMeta.Injection.COMPARE_FIELDS")
+ private List<CompareField> compareFields;
+
+ @HopMetadataProperty(
+ key = "reject_duplicate_row",
+ injectionKey = "REJECT_DUPLICATE_ROW",
+ injectionKeyDescription =
"UniqueRowsByHashSetMeta.Injection.REJECT_DUPLICATE_ROW")
private boolean rejectDuplicateRow;
+
+ @HopMetadataProperty(
+ key = "error_description",
+ injectionKey = "ERROR_DESCRIPTION",
+ injectionKeyDescription =
"UniqueRowsByHashSetMeta.Injection.ERROR_DESCRIPTION")
private String errorDescription;
public UniqueRowsByHashSetMeta() {
- super(); // allocate BaseTransformMeta
- }
-
- /**
- * @param compareField The compareField to set.
- */
- public void setCompareFields(String[] compareField) {
- this.compareFields = compareField;
- }
-
- public boolean getStoreValues() {
- return storeValues;
- }
-
- public void setStoreValues(boolean storeValues) {
- this.storeValues = storeValues;
- }
-
- /**
- * @return Returns the compareField.
- */
- public String[] getCompareFields() {
- return compareFields;
- }
-
- public void allocate(int nrFields) {
- compareFields = new String[nrFields];
- }
-
- /**
- * @param rejectDuplicateRow The rejectDuplicateRow to set.
- */
- public void setRejectDuplicateRow(boolean rejectDuplicateRow) {
- this.rejectDuplicateRow = rejectDuplicateRow;
- }
-
- /**
- * @return Returns the rejectDuplicateRow.
- */
- public boolean isRejectDuplicateRow() {
- return rejectDuplicateRow;
- }
-
- /**
- * @param errorDescription The errorDescription to set.
- */
- public void setErrorDescription(String errorDescription) {
- this.errorDescription = errorDescription;
- }
-
- /**
- * @return Returns the errorDescription.
- */
- public String getErrorDescription() {
- return errorDescription;
- }
-
- @Override
- public void loadXml(Node transformNode, IHopMetadataProvider
metadataProvider)
- throws HopXmlException {
- readData(transformNode);
- }
-
- @Override
- public Object clone() {
- UniqueRowsByHashSetMeta retval = (UniqueRowsByHashSetMeta) super.clone();
-
- int nrFields = compareFields.length;
-
- retval.allocate(nrFields);
-
- System.arraycopy(compareFields, 0, retval.compareFields, 0, nrFields);
- return retval;
- }
-
- private void readData(Node transformNode) throws HopXmlException {
- try {
- storeValues = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"store_values"));
- rejectDuplicateRow =
- "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"reject_duplicate_row"));
- errorDescription = XmlHandler.getTagValue(transformNode,
"error_description");
-
- Node fields = XmlHandler.getSubNode(transformNode, "fields");
- int nrFields = XmlHandler.countNodes(fields, CONST_FIELD);
-
- allocate(nrFields);
-
- for (int i = 0; i < nrFields; i++) {
- Node fnode = XmlHandler.getSubNodeByNr(fields, CONST_FIELD, i);
-
- compareFields[i] = XmlHandler.getTagValue(fnode, "name");
- }
-
- } catch (Exception e) {
- throw new HopXmlException(
- BaseMessages.getString(
- PKG,
"UniqueRowsByHashSetMeta.Exception.UnableToLoadTransformMetaFromXML"),
- e);
- }
- }
-
- @Override
- public void setDefault() {
+ super();
+ compareFields = new ArrayList<>();
rejectDuplicateRow = false;
errorDescription = null;
storeValues = true;
- int nrFields = 0;
-
- allocate(nrFields);
-
- for (int i = 0; i < nrFields; i++) {
- compareFields[i] = CONST_FIELD + i;
- }
}
- @Override
- public void getFields(
- IRowMeta row,
- String name,
- IRowMeta[] info,
- TransformMeta nextTransform,
- IVariables variables,
- IHopMetadataProvider metadataProvider)
- throws HopTransformException {
- // Do Nothing
+ public UniqueRowsByHashSetMeta(UniqueRowsByHashSetMeta m) {
+ this();
+ this.storeValues = m.storeValues;
+ this.rejectDuplicateRow = m.rejectDuplicateRow;
+ this.errorDescription = m.errorDescription;
+ m.compareFields.forEach(f -> this.compareFields.add(new CompareField(f)));
}
@Override
- public String getXml() {
- StringBuilder retval = new StringBuilder();
-
- retval.append(CONST_SPACES + XmlHandler.addTagValue("store_values",
storeValues));
- retval.append(
- CONST_SPACES + XmlHandler.addTagValue("reject_duplicate_row",
rejectDuplicateRow));
- retval.append(CONST_SPACES + XmlHandler.addTagValue("error_description",
errorDescription));
- retval.append(" <fields>");
- for (String compareField : compareFields) {
- retval.append(" <field>");
- retval.append(" " + XmlHandler.addTagValue("name", compareField));
- retval.append(" </field>");
- }
- retval.append(" </fields>");
-
- return retval.toString();
+ public Object clone() {
+ return new UniqueRowsByHashSetMeta(this);
}
@Override
diff --git
a/plugins/transforms/uniquerowsbyhashset/src/main/resources/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/messages/messages_en_US.properties
b/plugins/transforms/uniquerowsbyhashset/src/main/resources/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/messages/messages_en_US.properties
index 329a2f12bb..3d5cfcd8c0 100644
---
a/plugins/transforms/uniquerowsbyhashset/src/main/resources/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/messages/messages_en_US.properties
+++
b/plugins/transforms/uniquerowsbyhashset/src/main/resources/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/messages/messages_en_US.properties
@@ -36,3 +36,9 @@
UniqueRowsByHashSetMeta.CheckResult.NoInputReceivedFromOtherTransforms=No input
UniqueRowsByHashSetMeta.CheckResult.TransformReceivingInfoFromOtherTransforms=Transform
is receiving info from other transforms.
UniqueRowsByHashSetMeta.Exception.UnableToLoadTransformMetaFromXML=Unable to
load transform info from XML
UniqueRowsByHashSetMeta.keyword=unique,rows,hashset
+UniqueRowsByHashSetMeta.Injection.STORE_VALUES=Store values in memory or only
hash?
+UniqueRowsByHashSetMeta.Injection.COMPARE_FIELDS=The fields to compare for
uniqueness
+UniqueRowsByHashSetMeta.Injection.COMPARE_FIELD=A field to compare for
uniqueness
+UniqueRowsByHashSetMeta.Injection.REJECT_DUPLICATE_ROW=Reject duplicate rows?
+UniqueRowsByHashSetMeta.Injection.ERROR_DESCRIPTION=Rejected error description
+UniqueRowsByHashSetMeta.Injection.COMPARE_FIELD_NAME=Field name to compare for
uniqueness
\ No newline at end of file
diff --git
a/plugins/transforms/uniquerowsbyhashset/src/test/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetMetaTest.java
b/plugins/transforms/uniquerowsbyhashset/src/test/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetMetaTest.java
index 3be29962f7..6b3413c870 100644
---
a/plugins/transforms/uniquerowsbyhashset/src/test/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetMetaTest.java
+++
b/plugins/transforms/uniquerowsbyhashset/src/test/java/org/apache/hop/pipeline/transforms/uniquerowsbyhashset/UniqueRowsByHashSetMetaTest.java
@@ -17,57 +17,56 @@
package org.apache.hop.pipeline.transforms.uniquerowsbyhashset;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.hop.core.exception.HopException;
-import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
-import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.ArrayLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.StringLoadSaveValidator;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Objects;
+import org.apache.hop.core.xml.XmlHandler;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil;
+import org.apache.hop.pipeline.transform.TransformMeta;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
class UniqueRowsByHashSetMetaTest {
- @RegisterExtension
- static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
-
@Test
- void testRoundTrip() throws HopException {
- List<String> attributes =
- Arrays.asList("store_values", "reject_duplicate_row",
"error_description", "name");
-
- Map<String, String> getterMap = new HashMap<>();
- getterMap.put("store_values", "getStoreValues");
- getterMap.put("reject_duplicate_row", "isRejectDuplicateRow");
- getterMap.put("error_description", "getErrorDescription");
- getterMap.put("name", "getCompareFields");
-
- Map<String, String> setterMap = new HashMap<>();
- setterMap.put("store_values", "setStoreValues");
- setterMap.put("reject_duplicate_row", "setRejectDuplicateRow");
- setterMap.put("error_description", "setErrorDescription");
- setterMap.put("name", "setCompareFields");
+ void testLoadSave() throws Exception {
+ Path path =
Paths.get(Objects.requireNonNull(getClass().getResource("/transform.xml")).toURI());
+ String xml = Files.readString(path);
+ UniqueRowsByHashSetMeta meta = new UniqueRowsByHashSetMeta();
+ XmlMetadataUtil.deSerializeFromXml(
+ XmlHandler.loadXmlString(xml, TransformMeta.XML_TAG),
+ UniqueRowsByHashSetMeta.class,
+ meta,
+ new MemoryMetadataProvider());
- Map<String, IFieldLoadSaveValidator<?>> fieldLoadSaveValidatorAttributeMap
= new HashMap<>();
+ validate(meta);
- // Arrays need to be consistent length
- IFieldLoadSaveValidator<String[]> stringArrayLoadSaveValidator =
- new ArrayLoadSaveValidator<>(new StringLoadSaveValidator(), 25);
-
- fieldLoadSaveValidatorAttributeMap.put("name",
stringArrayLoadSaveValidator);
-
- LoadSaveTester loadSaveTester =
- new LoadSaveTester(
- UniqueRowsByHashSetMeta.class,
- attributes,
- getterMap,
- setterMap,
- fieldLoadSaveValidatorAttributeMap,
- new HashMap<>());
+ // Do a round trip:
+ //
+ String xmlCopy =
+ XmlHandler.openTag(TransformMeta.XML_TAG)
+ + XmlMetadataUtil.serializeObjectToXml(meta)
+ + XmlHandler.closeTag(TransformMeta.XML_TAG);
+ UniqueRowsByHashSetMeta metaCopy = new UniqueRowsByHashSetMeta();
+ XmlMetadataUtil.deSerializeFromXml(
+ XmlHandler.loadXmlString(xmlCopy, TransformMeta.XML_TAG),
+ UniqueRowsByHashSetMeta.class,
+ metaCopy,
+ new MemoryMetadataProvider());
+ validate(metaCopy);
+ }
- loadSaveTester.testSerialization();
+ private static void validate(UniqueRowsByHashSetMeta meta) {
+ assertNotNull(meta.getCompareFields());
+ assertTrue(meta.isStoreValues());
+ assertTrue(meta.isRejectDuplicateRow());
+ assertEquals("Duplicate row detected", meta.getErrorDescription());
+ assertEquals("compareField1", meta.getCompareFields().get(0).getName());
+ assertEquals("compareField2", meta.getCompareFields().get(1).getName());
+ assertEquals("compareField3", meta.getCompareFields().get(2).getName());
}
}
diff --git
a/plugins/transforms/uniquerowsbyhashset/src/test/resources/transform.xml
b/plugins/transforms/uniquerowsbyhashset/src/test/resources/transform.xml
new file mode 100644
index 0000000000..e81c2962ea
--- /dev/null
+++ b/plugins/transforms/uniquerowsbyhashset/src/test/resources/transform.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<transform>
+ <store_values>Y</store_values>
+ <reject_duplicate_row>Y</reject_duplicate_row>
+ <error_description>Duplicate row detected</error_description>
+ <fields>
+ <field>
+ <name>compareField1</name>
+ </field>
+ <field>
+ <name>compareField2</name>
+ </field>
+ <field>
+ <name>compareField3</name>
+ </field>
+ </fields>
+</transform>
