This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 572c3b8d9c Fix #5728 (#5733)
572c3b8d9c is described below
commit 572c3b8d9cc3079c47475a7c8128cfe7eff08e64
Author: lance <[email protected]>
AuthorDate: Wed Oct 1 19:48:06 2025 +0800
Fix #5728 (#5733)
* fix: Add sequence not transform execution copy safe
* fix: Add sequence not transform execution copy safe
* Add Sequence copy-safe integration tests
Signed-off-by: lance <[email protected]>
* fix hop/integration-tests/*/enhanced-json-output fail
Signed-off-by: lance <[email protected]>
* some cleanup and extra tests
more cleanup
---------
Signed-off-by: lance <[email protected]>
Signed-off-by: lance <[email protected]>
Co-authored-by: lance <[email protected]>
Co-authored-by: lance <[email protected]>
Co-authored-by: Hans Van Akelyen <[email protected]>
---
.../transforms/0001-add-sequence-copies.hpl | 63 ++-
...s.hpl => 0001-add-sequence-copy-safe-child.hpl} | 202 ++++-----
.../0001-add-sequence-copy-safe-parent.hpl | 153 +++++++
.../transforms/main-0001-add-sequence.hwf | 95 +++--
.../transforms/addsequence/AddSequence.java | 28 +-
.../transforms/addsequence/AddSequenceData.java | 36 +-
.../transforms/addsequence/AddSequenceMeta.java | 129 +-----
.../addsequence/AddSequenceDataTest.java | 80 ++++
.../addsequence/AddSequenceMetaTest.java | 106 ++++-
.../transforms/addsequence/AddSequenceTest.java | 468 +++++++++++++++++++++
.../transforms/jsonoutputenhanced/JsonOutput.java | 10 +-
11 files changed, 1000 insertions(+), 370 deletions(-)
diff --git a/integration-tests/transforms/0001-add-sequence-copies.hpl
b/integration-tests/transforms/0001-add-sequence-copies.hpl
index 1453dbd635..20c4cf2b90 100644
--- a/integration-tests/transforms/0001-add-sequence-copies.hpl
+++ b/integration-tests/transforms/0001-add-sequence-copies.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>00100-add-sequence-copies</name>
+ <name>0001-add-sequence-copies</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -40,8 +40,6 @@ limitations under the License.
<created_date>2020/12/16 12:25:44.129</created_date>
<modified_user>-</modified_user>
<modified_date>2020/12/16 12:25:44.129</modified_date>
- <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
- <is_key_private>N</is_key_private>
</info>
<notepads>
</notepads>
@@ -84,12 +82,12 @@ limitations under the License.
<schema_name/>
</partitioning>
<fields>
- </fields>
+</fields>
+ <interval_in_ms>5000</interval_in_ms>
+ <last_time_field>FiveSecondsAgo</last_time_field>
<limit>${COUNT}</limit>
<never_ending>N</never_ending>
- <interval_in_ms>5000</interval_in_ms>
<row_time_field>now</row_time_field>
- <last_time_field>FiveSecondsAgo</last_time_field>
<attributes/>
<GUI>
<xloc>128</xloc>
@@ -107,10 +105,10 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <row_threshold>0</row_threshold>
- <message>Transform Add Sequence didn't generate the expected IDs running
in a multiple copies</message>
- <always_log_rows>Y</always_log_rows>
<abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <message>Transform Add Sequence didn't generate the expected IDs running
in a multiple copies</message>
+ <row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>736</xloc>
@@ -128,24 +126,21 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
+ <add_linenr>N</add_linenr>
<all_rows>N</all_rows>
- <ignore_aggregate>N</ignore_aggregate>
- <field_ignore/>
<directory>${java.io.tmpdir}</directory>
- <prefix>grp</prefix>
- <add_linenr>N</add_linenr>
- <linenr_fieldname/>
- <give_back_row>N</give_back_row>
- <group>
- </group>
<fields>
<field>
<aggregate>count</aggregate>
<subject>id</subject>
<type>COUNT_DISTINCT</type>
- <valuefield/>
</field>
</fields>
+ <give_back_row>N</give_back_row>
+ <group>
+</group>
+ <ignore_aggregate>N</ignore_aggregate>
+ <prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>400</xloc>
@@ -165,16 +160,12 @@ limitations under the License.
</partitioning>
<fields>
<field>
- <name>expectedCount</name>
- <variable>${COUNT}</variable>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
<length>-1</length>
+ <name>expectedCount</name>
<precision>-1</precision>
<trim_type>none</trim_type>
+ <type>String</type>
+ <variable>${COUNT}</variable>
</field>
</fields>
<attributes/>
@@ -194,16 +185,13 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <valuename>id</valuename>
- <use_database>N</use_database>
- <connection/>
- <schema/>
- <seqname>SEQ_</seqname>
- <use_counter>Y</use_counter>
- <counter_name/>
- <start_at>1</start_at>
<increment_by>1</increment_by>
<max_value>999999999</max_value>
+ <seqname>SEQ_</seqname>
+ <start_at>1</start_at>
+ <use_counter>Y</use_counter>
+ <use_database>N</use_database>
+ <valuename>id</valuename>
<attributes/>
<GUI>
<xloc>256</xloc>
@@ -221,13 +209,14 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <send_true_to/>
- <send_false_to/>
<compare>
<condition>
- <negated>N</negated>
- <leftvalue>count</leftvalue>
+ <conditions>
+</conditions>
<function><></function>
+ <leftvalue>count</leftvalue>
+ <negated>N</negated>
+ <operator>-</operator>
<rightvalue>expectedCount</rightvalue>
</condition>
</compare>
diff --git a/integration-tests/transforms/0001-add-sequence-copies.hpl
b/integration-tests/transforms/0001-add-sequence-copy-safe-child.hpl
similarity index 53%
copy from integration-tests/transforms/0001-add-sequence-copies.hpl
copy to integration-tests/transforms/0001-add-sequence-copy-safe-child.hpl
index 1453dbd635..83c73f0592 100644
--- a/integration-tests/transforms/0001-add-sequence-copies.hpl
+++ b/integration-tests/transforms/0001-add-sequence-copy-safe-child.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>00100-add-sequence-copies</name>
+ <name>0001-add-sequence-copy-safe-child</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -28,77 +28,51 @@ limitations under the License.
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
- <name>COUNT</name>
- <default_value>1000000</default_value>
- <description>Number of IDs to generate</description>
+ <name>PARAM_INCREMENT_VALUE</name>
+ <default_value/>
+ <description/>
+ </parameter>
+ <parameter>
+ <name>PARAM_INSTANCE</name>
+ <default_value/>
+ <description/>
+ </parameter>
+ <parameter>
+ <name>PARAM_START_VALUE</name>
+ <default_value/>
+ <description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
- <created_date>2020/12/16 12:25:44.129</created_date>
+ <created_date>2025/09/23 09:45:31.937</created_date>
<modified_user>-</modified_user>
- <modified_date>2020/12/16 12:25:44.129</modified_date>
- <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
- <is_key_private>N</is_key_private>
+ <modified_date>2025/09/23 09:45:31.937</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
- <from>${COUNT} rows</from>
- <to>id</to>
- <enabled>Y</enabled>
- </hop>
- <hop>
- <from>not expected?</from>
- <to>Abort</to>
- <enabled>Y</enabled>
- </hop>
- <hop>
- <from>id</from>
- <to>count distinct</to>
+ <from>Generate rows</from>
+ <to>Get PARAMs</to>
<enabled>Y</enabled>
</hop>
<hop>
- <from>count distinct</from>
- <to>expected count</to>
+ <from>Get PARAMs</from>
+ <to>Add sequence</to>
<enabled>Y</enabled>
</hop>
<hop>
- <from>expected count</from>
- <to>not expected?</to>
+ <from>Add sequence</from>
+ <to>Write to log</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
- <name>${COUNT} rows</name>
- <type>RowGenerator</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <fields>
- </fields>
- <limit>${COUNT}</limit>
- <never_ending>N</never_ending>
- <interval_in_ms>5000</interval_in_ms>
- <row_time_field>now</row_time_field>
- <last_time_field>FiveSecondsAgo</last_time_field>
- <attributes/>
- <GUI>
- <xloc>128</xloc>
- <yloc>96</yloc>
- </GUI>
- </transform>
- <transform>
- <name>Abort</name>
- <type>Abort</type>
+ <name>Add sequence</name>
+ <type>Sequence</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@@ -107,19 +81,22 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <row_threshold>0</row_threshold>
- <message>Transform Add Sequence didn't generate the expected IDs running
in a multiple copies</message>
- <always_log_rows>Y</always_log_rows>
- <abort_option>ABORT_WITH_ERROR</abort_option>
+ <increment_by>1</increment_by>
+ <max_value>999999999</max_value>
+ <seqname>SEQ_</seqname>
+ <start_at>${PARAM_START_VALUE}</start_at>
+ <use_counter>Y</use_counter>
+ <use_database>N</use_database>
+ <valuename>sequence_value</valuename>
<attributes/>
<GUI>
- <xloc>736</xloc>
- <yloc>96</yloc>
+ <xloc>848</xloc>
+ <yloc>368</yloc>
</GUI>
</transform>
<transform>
- <name>count distinct</name>
- <type>GroupBy</type>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@@ -128,32 +105,21 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <all_rows>N</all_rows>
- <ignore_aggregate>N</ignore_aggregate>
- <field_ignore/>
- <directory>${java.io.tmpdir}</directory>
- <prefix>grp</prefix>
- <add_linenr>N</add_linenr>
- <linenr_fieldname/>
- <give_back_row>N</give_back_row>
- <group>
- </group>
<fields>
- <field>
- <aggregate>count</aggregate>
- <subject>id</subject>
- <type>COUNT_DISTINCT</type>
- <valuefield/>
- </field>
- </fields>
+</fields>
+ <interval_in_ms>5000</interval_in_ms>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <limit>50</limit>
+ <never_ending>N</never_ending>
+ <row_time_field>now</row_time_field>
<attributes/>
<GUI>
- <xloc>400</xloc>
- <yloc>96</yloc>
+ <xloc>464</xloc>
+ <yloc>368</yloc>
</GUI>
</transform>
<transform>
- <name>expected count</name>
+ <name>Get PARAMs</name>
<type>GetVariable</type>
<description/>
<distribute>Y</distribute>
@@ -165,54 +131,31 @@ limitations under the License.
</partitioning>
<fields>
<field>
- <name>expectedCount</name>
- <variable>${COUNT}</variable>
+ <length>-1</length>
+ <name>param_instance</name>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
<type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
+ <variable>${PARAM_INSTANCE}</variable>
+ </field>
+ <field>
<length>-1</length>
+ <name>param_start_value</name>
<precision>-1</precision>
<trim_type>none</trim_type>
+ <type>Integer</type>
+ <variable>${PARAM_START_VALUE}</variable>
</field>
</fields>
<attributes/>
<GUI>
- <xloc>512</xloc>
- <yloc>96</yloc>
+ <xloc>672</xloc>
+ <yloc>368</yloc>
</GUI>
</transform>
<transform>
- <name>id</name>
- <type>Sequence</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>10</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <valuename>id</valuename>
- <use_database>N</use_database>
- <connection/>
- <schema/>
- <seqname>SEQ_</seqname>
- <use_counter>Y</use_counter>
- <counter_name/>
- <start_at>1</start_at>
- <increment_by>1</increment_by>
- <max_value>999999999</max_value>
- <attributes/>
- <GUI>
- <xloc>256</xloc>
- <yloc>96</yloc>
- </GUI>
- </transform>
- <transform>
- <name>not expected?</name>
- <type>FilterRows</type>
+ <name>Write to log</name>
+ <type>WriteToLog</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@@ -221,20 +164,25 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <send_true_to/>
- <send_false_to/>
- <compare>
- <condition>
- <negated>N</negated>
- <leftvalue>count</leftvalue>
- <function><></function>
- <rightvalue>expectedCount</rightvalue>
- </condition>
- </compare>
+ <displayHeader>Y</displayHeader>
+ <fields>
+ <field>
+ <name>param_instance</name>
+ </field>
+ <field>
+ <name>param_start_value</name>
+ </field>
+ <field>
+ <name>sequence_value</name>
+ </field>
+ </fields>
+ <limitRows>N</limitRows>
+ <limitRowsNumber>0</limitRowsNumber>
+ <loglevel>Basic</loglevel>
<attributes/>
<GUI>
- <xloc>624</xloc>
- <yloc>96</yloc>
+ <xloc>1072</xloc>
+ <yloc>368</yloc>
</GUI>
</transform>
<transform_error_handling>
diff --git
a/integration-tests/transforms/0001-add-sequence-copy-safe-parent.hpl
b/integration-tests/transforms/0001-add-sequence-copy-safe-parent.hpl
new file mode 100644
index 0000000000..593d6ba660
--- /dev/null
+++ b/integration-tests/transforms/0001-add-sequence-copy-safe-parent.hpl
@@ -0,0 +1,153 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0001-add-sequence-copy-safe-parent</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2025/09/23 09:42:54.496</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/09/23 09:42:54.496</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Data grid</from>
+ <to>0001-add-sequence-copy-safe-child.hpl</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>0001-add-sequence-copy-safe-child.hpl</name>
+ <type>PipelineExecutor</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>2</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <execution_errors_field>ExecutionNrErrors</execution_errors_field>
+
<execution_exit_status_field>ExecutionExitStatus</execution_exit_status_field>
+
<execution_files_retrieved_field>ExecutionFilesRetrieved</execution_files_retrieved_field>
+
<execution_lines_deleted_field>ExecutionLinesDeleted</execution_lines_deleted_field>
+
<execution_lines_input_field>ExecutionLinesInput</execution_lines_input_field>
+
<execution_lines_output_field>ExecutionLinesOutput</execution_lines_output_field>
+ <execution_lines_read_field>ExecutionLinesRead</execution_lines_read_field>
+
<execution_lines_rejected_field>ExecutionLinesRejected</execution_lines_rejected_field>
+
<execution_lines_updated_field>ExecutionLinesUpdated</execution_lines_updated_field>
+
<execution_lines_written_field>ExecutionLinesWritten</execution_lines_written_field>
+
<execution_log_channelid_field>ExecutionLogChannelId</execution_log_channelid_field>
+ <execution_log_text_field>ExecutionLogText</execution_log_text_field>
+ <execution_result_field>ExecutionResult</execution_result_field>
+ <execution_result_target_transform/>
+ <execution_time_field>ExecutionTime</execution_time_field>
+ <filename>${PROJECT_HOME}/0001-add-sequence-copy-safe-child.hpl</filename>
+ <filenameField/>
+ <filenameInField>N</filenameInField>
+ <group_field/>
+ <group_size>1</group_size>
+ <group_time/>
+ <inherit_all_vars>N</inherit_all_vars>
+ <parameters>
+ <variable_mapping>
+ <field>increment_value</field>
+ <input/>
+ <variable>PARAM_INCREMENT_VALUE</variable>
+ </variable_mapping>
+ <variable_mapping>
+ <field>instance</field>
+ <input/>
+ <variable>PARAM_INSTANCE</variable>
+ </variable_mapping>
+ <variable_mapping>
+ <field>start_value</field>
+ <input/>
+ <variable>PARAM_START_VALUE</variable>
+ </variable_mapping>
+ </parameters>
+ <result_files_file_name_field>FileName</result_files_file_name_field>
+ <result_files_target_transform/>
+ <result_rows_target_transform/>
+ <run_configuration>local</run_configuration>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>288</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Data grid</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>instance 1</item>
+ </line>
+ <line>
+ <item>666667</item>
+ <item>instance 666667</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>start_value</name>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>instance</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>336</xloc>
+ <yloc>288</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/main-0001-add-sequence.hwf
b/integration-tests/transforms/main-0001-add-sequence.hwf
index ac9676f87a..c2b49e314d 100644
--- a/integration-tests/transforms/main-0001-add-sequence.hwf
+++ b/integration-tests/transforms/main-0001-add-sequence.hwf
@@ -36,16 +36,16 @@ limitations under the License.
<description/>
<type>SPECIAL</type>
<attributes/>
- <repeat>N</repeat>
- <schedulerType>0</schedulerType>
- <intervalSeconds>0</intervalSeconds>
- <intervalMinutes>60</intervalMinutes>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
<hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
<minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
<weekDay>1</weekDay>
- <DayOfMonth>1</DayOfMonth>
<parallel>N</parallel>
- <nr>0</nr>
<xloc>128</xloc>
<yloc>96</yloc>
<attributes_hac/>
@@ -55,27 +55,23 @@ limitations under the License.
<description/>
<type>PIPELINE</type>
<attributes/>
- <filename>${PROJECT_HOME}/0001-add-sequence-single.hpl</filename>
- <params_from_previous>N</params_from_previous>
- <exec_per_row>N</exec_per_row>
- <clear_rows>N</clear_rows>
- <clear_files>N</clear_files>
- <set_logfile>N</set_logfile>
- <logfile/>
- <logext/>
<add_date>N</add_date>
<add_time>N</add_time>
- <loglevel>Basic</loglevel>
- <set_append_logfile>N</set_append_logfile>
- <wait_until_finished>Y</wait_until_finished>
- <follow_abort_remote>N</follow_abort_remote>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
- <run_configuration>local</run_configuration>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0001-add-sequence-single.hpl</filename>
+ <loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
- <nr>0</nr>
<xloc>304</xloc>
<yloc>96</yloc>
<attributes_hac/>
@@ -85,28 +81,52 @@ limitations under the License.
<description/>
<type>PIPELINE</type>
<attributes/>
- <filename>${PROJECT_HOME}/0001-add-sequence-copies.hpl</filename>
- <params_from_previous>N</params_from_previous>
- <exec_per_row>N</exec_per_row>
- <clear_rows>N</clear_rows>
- <clear_files>N</clear_files>
- <set_logfile>N</set_logfile>
- <logfile/>
- <logext/>
<add_date>N</add_date>
<add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0001-add-sequence-copies.hpl</filename>
<loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
- <follow_abort_remote>N</follow_abort_remote>
+ <parallel>N</parallel>
+ <xloc>528</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0001-add-sequence-copy-safe-parent.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
- <run_configuration>local</run_configuration>
+ <exec_per_row>N</exec_per_row>
+
<filename>${PROJECT_HOME}/0001-add-sequence-copy-safe-parent.hpl</filename>
+ <logext/>
+ <logfile/>
+ <loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
- <nr>0</nr>
- <xloc>528</xloc>
+ <xloc>832</xloc>
<yloc>96</yloc>
<attributes_hac/>
</action>
@@ -115,8 +135,6 @@ limitations under the License.
<hop>
<from>Start</from>
<to>0001-add-sequence-single.hpl</to>
- <from_nr>0</from_nr>
- <to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
@@ -124,8 +142,13 @@ limitations under the License.
<hop>
<from>0001-add-sequence-single.hpl</from>
<to>0001-add-sequence-copies.hpl</to>
- <from_nr>0</from_nr>
- <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0001-add-sequence-copies.hpl</from>
+ <to>0001-add-sequence-copy-safe-parent.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
diff --git
a/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequence.java
b/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequence.java
index 7d75cce0bb..9f442de636 100644
---
a/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequence.java
+++
b/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequence.java
@@ -56,6 +56,9 @@ public class AddSequence extends
BaseTransform<AddSequenceMeta, AddSequenceData>
if (meta.isCounterUsed()) {
next = data.counter.getAndNext();
+ if (isDetailed()) {
+ logDetailed("count name: {0}, next: {1}", data.getLookup(), next);
+ }
} else if (meta.isDatabaseUsed()) {
try {
next =
@@ -90,8 +93,8 @@ public class AddSequence extends
BaseTransform<AddSequenceMeta, AddSequenceData>
@Override
public boolean processRow() throws HopException {
-
- Object[] r = getRow(); // Get row from input rowset & set row busy!
+ // Get row from input rowset & set row busy!
+ Object[] r = getRow();
if (r == null) {
// no more input to be expected...
setOutputDone();
@@ -216,9 +219,9 @@ public class AddSequence extends
BaseTransform<AddSequenceMeta, AddSequenceData>
String realCounterName = resolve(meta.getCounterName());
if (!Utils.isEmpty(realCounterName)) {
- data.setLookup("@@sequence:" + realCounterName);
+ data.setLookup(lookupCounterName(realCounterName));
} else {
- data.setLookup("@@sequence:" + meta.getValueName());
+ data.setLookup(lookupCounterName(meta.getValueName()));
}
// We need to synchronize over the whole pipeline to make sure that we
always get the same
@@ -226,6 +229,7 @@ public class AddSequence extends
BaseTransform<AddSequenceMeta, AddSequenceData>
// regardless of the number of transform copies asking for it.
//
synchronized (getPipeline()) {
+ logBasic("init counter name: {0}", data.getLookup());
data.counter =
Counters.getInstance()
.getOrUpdateCounter(
@@ -264,4 +268,20 @@ public class AddSequence extends
BaseTransform<AddSequenceMeta, AddSequenceData>
public void cleanup() {
super.cleanup();
}
+
+ /**
+ * Build a unique identifier for this pipeline run.
+ *
+ * @param counterName counter name
+ * @return unique key
+ */
+ private String lookupCounterName(String counterName) {
+ // unique per run
+ String scope = getPipeline().getContainerId();
+ if (Utils.isEmpty(scope)) {
+ // fallback: unique per run as well
+ scope = getPipeline().getLogChannelId();
+ }
+ return "@@sequence:" + scope + ":" + counterName;
+ }
}
diff --git
a/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceData.java
b/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceData.java
index 9be9d21c82..afa4ea280c 100644
---
a/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceData.java
+++
b/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceData.java
@@ -17,6 +17,8 @@
package org.apache.hop.pipeline.transforms.addsequence;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.Counter;
import org.apache.hop.core.database.Database;
import org.apache.hop.core.row.IRowMeta;
@@ -25,8 +27,9 @@ import org.apache.hop.pipeline.transform.ITransformData;
@SuppressWarnings("java:S1104")
public class AddSequenceData extends BaseTransformData implements
ITransformData {
- private Database db;
- private String lookup;
+ @Setter @Getter private Database db;
+
+ @Setter @Getter private String lookup;
public IRowMeta outputRowMeta;
public Counter counter;
@@ -38,7 +41,6 @@ public class AddSequenceData extends BaseTransformData
implements ITransformData
public String realSchemaName;
public String realSequenceName;
- /** */
public AddSequenceData() {
super();
@@ -46,32 +48,4 @@ public class AddSequenceData extends BaseTransformData
implements ITransformData
realSchemaName = null;
realSequenceName = null;
}
-
- /**
- * @return Returns the db.
- */
- public Database getDb() {
- return db;
- }
-
- /**
- * @param db The db to set.
- */
- public void setDb(Database db) {
- this.db = db;
- }
-
- /**
- * @return Returns the lookup string usually "@@"+the name of the sequence.
- */
- public String getLookup() {
- return lookup;
- }
-
- /**
- * @param lookup the lookup string usually "@@"+the name of the sequence.
- */
- public void setLookup(String lookup) {
- this.lookup = lookup;
- }
}
diff --git
a/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java
b/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java
index add6002f09..9b37505c7f 100644
---
a/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java
+++
b/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java
@@ -18,6 +18,8 @@
package org.apache.hop.pipeline.transforms.addsequence;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.Const;
import org.apache.hop.core.ICheckResult;
@@ -46,6 +48,8 @@ import org.apache.hop.pipeline.transform.TransformMeta;
categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform",
documentationUrl = "/pipeline/transforms/addsequence.html",
keywords = "i18n::AddSequenceMeta.keyword")
+@Getter
+@Setter
public class AddSequenceMeta extends BaseTransformMeta<AddSequence,
AddSequenceData> {
private static final Class<?> PKG = AddSequenceMeta.class;
@@ -100,33 +104,11 @@ public class AddSequenceMeta extends
BaseTransformMeta<AddSequence, AddSequenceD
injectionKeyDescription = "AddSequenceMeta.Injection.MaxValue")
private String maxValue;
- public String getConnection() {
- return connection;
- }
-
- public void setConnection(String connection) {
- this.connection = connection;
- }
-
- /**
- * @return Returns the incrementBy.
- */
- public String getIncrementBy() {
- return incrementBy;
- }
-
/**
- * @param incrementBy The incrementBy to set.
- */
- public void setIncrementBy(String incrementBy) {
- this.incrementBy = incrementBy;
- }
-
- /**
- * @return Returns the maxValue.
+ * @param maxValue The maxValue to set.
*/
- public String getMaxValue() {
- return maxValue;
+ public void setMaxValue(long maxValue) {
+ this.maxValue = Long.toString(maxValue);
}
/**
@@ -136,27 +118,6 @@ public class AddSequenceMeta extends
BaseTransformMeta<AddSequence, AddSequenceD
this.maxValue = maxValue;
}
- /**
- * @return Returns the sequenceName.
- */
- public String getSequenceName() {
- return sequenceName;
- }
-
- /**
- * @param sequenceName The sequenceName to set.
- */
- public void setSequenceName(String sequenceName) {
- this.sequenceName = sequenceName;
- }
-
- /**
- * @param maxValue The maxValue to set.
- */
- public void setMaxValue(long maxValue) {
- this.maxValue = Long.toString(maxValue);
- }
-
/**
* @param startAt The starting point of the sequence to set.
*/
@@ -172,10 +133,10 @@ public class AddSequenceMeta extends
BaseTransformMeta<AddSequence, AddSequenceD
}
/**
- * @return Returns the start of the sequence.
+ * @param incrementBy The incrementBy to set.
*/
- public String getStartAt() {
- return startAt;
+ public void setIncrementBy(String incrementBy) {
+ this.incrementBy = incrementBy;
}
/**
@@ -185,48 +146,6 @@ public class AddSequenceMeta extends
BaseTransformMeta<AddSequence, AddSequenceD
this.startAt = startAt;
}
- /**
- * @return Returns the useCounter.
- */
- public boolean isCounterUsed() {
- return counterUsed;
- }
-
- /**
- * @param counterUsed The useCounter to set.
- */
- public void setCounterUsed(boolean counterUsed) {
- this.counterUsed = counterUsed;
- }
-
- /**
- * @return Returns the useDatabase.
- */
- public boolean isDatabaseUsed() {
- return databaseUsed;
- }
-
- /**
- * @param databaseUsed The useDatabase to set.
- */
- public void setDatabaseUsed(boolean databaseUsed) {
- this.databaseUsed = databaseUsed;
- }
-
- /**
- * @return Returns the valuename.
- */
- public String getValueName() {
- return valueName;
- }
-
- /**
- * @param valueName The valuename to set.
- */
- public void setValueName(String valueName) {
- this.valueName = valueName;
- }
-
@Override
public Object clone() {
return super.clone();
@@ -377,32 +296,4 @@ public class AddSequenceMeta extends
BaseTransformMeta<AddSequence, AddSequenceD
return retval;
}
-
- /**
- * @return the counterName
- */
- public String getCounterName() {
- return counterName;
- }
-
- /**
- * @param counterName the counterName to set
- */
- public void setCounterName(String counterName) {
- this.counterName = counterName;
- }
-
- /**
- * @return the schemaName
- */
- public String getSchemaName() {
- return schemaName;
- }
-
- /**
- * @param schemaName the schemaName to set
- */
- public void setSchemaName(String schemaName) {
- this.schemaName = schemaName;
- }
}
diff --git
a/plugins/transforms/addsequence/src/test/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceDataTest.java
b/plugins/transforms/addsequence/src/test/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceDataTest.java
new file mode 100644
index 0000000000..00d647c6b1
--- /dev/null
+++
b/plugins/transforms/addsequence/src/test/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceDataTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.addsequence;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.Mockito.mock;
+
+import org.apache.hop.core.Counter;
+import org.apache.hop.core.database.Database;
+import org.apache.hop.core.row.IRowMeta;
+import org.junit.jupiter.api.Test;
+
+class AddSequenceDataTest {
+
+ @Test
+ void testDefaultConstructor() {
+ AddSequenceData data = new AddSequenceData();
+
+ assertNull(data.getDb());
+ assertNull(data.getLookup());
+ assertNull(data.outputRowMeta);
+ assertNull(data.counter);
+ assertEquals(0L, data.start);
+ assertEquals(0L, data.increment);
+ assertEquals(0L, data.maximum);
+ assertNull(data.realSchemaName);
+ assertNull(data.realSequenceName);
+ }
+
+ @Test
+ void testSettersAndGetters() {
+ AddSequenceData data = new AddSequenceData();
+
+ Database db = mock(Database.class);
+ data.setDb(db);
+ assertEquals(db, data.getDb());
+
+ data.setLookup("@@sequence:test");
+ assertEquals("@@sequence:test", data.getLookup());
+
+ Counter counter = mock(Counter.class);
+ data.counter = counter;
+ assertEquals(counter, data.counter);
+
+ IRowMeta rowMeta = mock(IRowMeta.class);
+ data.outputRowMeta = rowMeta;
+ assertEquals(rowMeta, data.outputRowMeta);
+
+ data.start = 10L;
+ assertEquals(10L, data.start);
+
+ data.increment = 2L;
+ assertEquals(2L, data.increment);
+
+ data.maximum = 100L;
+ assertEquals(100L, data.maximum);
+
+ data.realSchemaName = "public";
+ assertEquals("public", data.realSchemaName);
+
+ data.realSequenceName = "my_seq";
+ assertEquals("my_seq", data.realSequenceName);
+ }
+}
diff --git
a/plugins/transforms/addsequence/src/test/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMetaTest.java
b/plugins/transforms/addsequence/src/test/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMetaTest.java
index a4396a5a6f..a72a3f6ee4 100644
---
a/plugins/transforms/addsequence/src/test/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMetaTest.java
+++
b/plugins/transforms/addsequence/src/test/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMetaTest.java
@@ -17,10 +17,15 @@
package org.apache.hop.pipeline.transforms.addsequence;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.apache.hop.core.HopEnvironment;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
@@ -41,7 +46,7 @@ class AddSequenceMetaTest {
}
@Test
- void testRoundTrip() throws HopException {
+ void testSerialization() throws HopException {
List<String> attributes =
Arrays.asList(
"valueName",
@@ -55,20 +60,93 @@ class AddSequenceMetaTest {
"incrementBy",
"maxValue");
- Map<String, String> getterMap = new HashMap<>();
- Map<String, String> setterMap = new HashMap<>();
-
- Map<String, IFieldLoadSaveValidator<?>> typeValidators = new HashMap<>();
- Map<String, IFieldLoadSaveValidator<?>> fieldValidators = new HashMap<>();
-
- LoadSaveTester loadSaveTester =
- new LoadSaveTester(
+ LoadSaveTester<AddSequenceMeta> loadSaveTester =
+ new LoadSaveTester<>(
AddSequenceMeta.class,
attributes,
- getterMap,
- setterMap,
- fieldValidators,
- typeValidators);
+ new HashMap<>(),
+ new HashMap<>(),
+ new HashMap<String, IFieldLoadSaveValidator<?>>(),
+ new HashMap<String, IFieldLoadSaveValidator<?>>());
loadSaveTester.testSerialization();
}
+
+ @Test
+ void testSetDefault() {
+ AddSequenceMeta meta = new AddSequenceMeta();
+ meta.setDefault();
+
+ assertEquals("valuename", meta.getValueName());
+ assertFalse(meta.isDatabaseUsed());
+ assertTrue(meta.isCounterUsed());
+ assertEquals("", meta.getSchemaName());
+ assertEquals("SEQ_", meta.getSequenceName());
+ assertNull(meta.getCounterName());
+ assertEquals("1", meta.getStartAt());
+ assertEquals("1", meta.getIncrementBy());
+ assertEquals("999999999", meta.getMaxValue());
+ }
+
+ @Test
+ void testGettersAndSetters() {
+ AddSequenceMeta meta = new AddSequenceMeta();
+
+ meta.setValueName("test_seq");
+ assertEquals("test_seq", meta.getValueName());
+
+ meta.setDatabaseUsed(true);
+ assertTrue(meta.isDatabaseUsed());
+
+ meta.setCounterUsed(false);
+ assertFalse(meta.isCounterUsed());
+
+ meta.setConnection("db_conn");
+ assertEquals("db_conn", meta.getConnection());
+
+ meta.setSchemaName("schema1");
+ assertEquals("schema1", meta.getSchemaName());
+
+ meta.setSequenceName("seq1");
+ assertEquals("seq1", meta.getSequenceName());
+
+ meta.setCounterName("counter1");
+ assertEquals("counter1", meta.getCounterName());
+
+ // Test string setters
+ meta.setStartAt("10");
+ assertEquals("10", meta.getStartAt());
+
+ meta.setIncrementBy("5");
+ assertEquals("5", meta.getIncrementBy());
+
+ meta.setMaxValue("1000");
+ assertEquals("1000", meta.getMaxValue());
+
+ // Test long setters
+ meta.setStartAt(100L);
+ assertEquals("100", meta.getStartAt());
+
+ meta.setIncrementBy(10L);
+ assertEquals("10", meta.getIncrementBy());
+
+ meta.setMaxValue(10000L);
+ assertEquals("10000", meta.getMaxValue());
+ }
+
+ @Test
+ void testClone() {
+ AddSequenceMeta meta = new AddSequenceMeta();
+ meta.setValueName("test_value");
+ meta.setDatabaseUsed(true);
+ meta.setConnection("test_connection");
+ meta.setStartAt("100");
+
+ AddSequenceMeta cloned = (AddSequenceMeta) meta.clone();
+
+ assertNotNull(cloned);
+ assertEquals(meta.getValueName(), cloned.getValueName());
+ assertEquals(meta.isDatabaseUsed(), cloned.isDatabaseUsed());
+ assertEquals(meta.getConnection(), cloned.getConnection());
+ assertEquals(meta.getStartAt(), cloned.getStartAt());
+ }
}
diff --git
a/plugins/transforms/addsequence/src/test/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceTest.java
b/plugins/transforms/addsequence/src/test/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceTest.java
new file mode 100644
index 0000000000..b280cca56c
--- /dev/null
+++
b/plugins/transforms/addsequence/src/test/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceTest.java
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.addsequence;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import org.apache.hop.core.Counters;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.database.Database;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.PipelineTestingUtil;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+class AddSequenceTest {
+
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ private TransformMockHelper<AddSequenceMeta, AddSequenceData>
transformMockHelper;
+
+ @BeforeAll
+ static void setUpBeforeClass() throws HopException {
+ HopEnvironment.init();
+ }
+
+ @BeforeEach
+ void setUp() {
+ transformMockHelper =
+ new TransformMockHelper<>("AddSequence", AddSequenceMeta.class,
AddSequenceData.class);
+ when(transformMockHelper.logChannelFactory.create(any(),
any(ILoggingObject.class)))
+ .thenReturn(transformMockHelper.iLogChannel);
+ when(transformMockHelper.pipeline.isRunning()).thenReturn(true);
+
when(transformMockHelper.transformMeta.getName()).thenReturn("AddSequence");
+ }
+
+ @AfterEach
+ void tearDown() {
+ // Clean up all counters to prevent test pollution
+ try {
+ Counters.getInstance().clear();
+ } catch (Exception e) {
+ // Ignore cleanup errors
+ }
+ transformMockHelper.cleanUp();
+ }
+
+ /** Test init() method with counter configuration */
+ @Test
+ void testInitWithCounter() {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(true);
+
when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(false);
+
when(transformMockHelper.iTransformMeta.getValueName()).thenReturn("test_seq");
+ when(transformMockHelper.iTransformMeta.getStartAt()).thenReturn("5");
+ when(transformMockHelper.iTransformMeta.getIncrementBy()).thenReturn("2");
+ when(transformMockHelper.iTransformMeta.getMaxValue()).thenReturn("1000");
+ when(transformMockHelper.pipeline.getContainerId())
+ .thenReturn("test-" + System.currentTimeMillis());
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ boolean result = addSequence.init();
+
+ assertTrue(result);
+ assertNotNull(transformMockHelper.iTransformData.counter);
+ assertEquals(5L, transformMockHelper.iTransformData.start);
+ assertEquals(2L, transformMockHelper.iTransformData.increment);
+ assertEquals(1000L, transformMockHelper.iTransformData.maximum);
+
+ addSequence.dispose();
+ }
+
+ /** Test init() method with invalid start value */
+ @Test
+ void testInitWithInvalidStartValue() {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(true);
+
when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(false);
+
when(transformMockHelper.iTransformMeta.getValueName()).thenReturn("test_seq");
+
when(transformMockHelper.iTransformMeta.getStartAt()).thenReturn("invalid");
+ when(transformMockHelper.iTransformMeta.getIncrementBy()).thenReturn("2");
+ when(transformMockHelper.iTransformMeta.getMaxValue()).thenReturn("1000");
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ boolean result = addSequence.init();
+
+ assertFalse(result);
+ }
+
+ /** Test init() method with invalid increment value */
+ @Test
+ void testInitWithInvalidIncrementValue() {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(true);
+
when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(false);
+
when(transformMockHelper.iTransformMeta.getValueName()).thenReturn("test_seq");
+ when(transformMockHelper.iTransformMeta.getStartAt()).thenReturn("1");
+
when(transformMockHelper.iTransformMeta.getIncrementBy()).thenReturn("invalid");
+ when(transformMockHelper.iTransformMeta.getMaxValue()).thenReturn("1000");
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ boolean result = addSequence.init();
+
+ assertFalse(result);
+ }
+
+ /** Test init() method with invalid max value */
+ @Test
+ void testInitWithInvalidMaxValue() {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(true);
+
when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(false);
+
when(transformMockHelper.iTransformMeta.getValueName()).thenReturn("test_seq");
+ when(transformMockHelper.iTransformMeta.getStartAt()).thenReturn("1");
+ when(transformMockHelper.iTransformMeta.getIncrementBy()).thenReturn("1");
+
when(transformMockHelper.iTransformMeta.getMaxValue()).thenReturn("invalid");
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ boolean result = addSequence.init();
+
+ assertFalse(result);
+ }
+
+ /** Test dispose() method cleans up counter resources */
+ @Test
+ void testDisposeWithCounter() {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(true);
+
when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(false);
+
when(transformMockHelper.iTransformMeta.getValueName()).thenReturn("test_seq");
+ when(transformMockHelper.iTransformMeta.getStartAt()).thenReturn("1");
+ when(transformMockHelper.iTransformMeta.getIncrementBy()).thenReturn("1");
+ when(transformMockHelper.iTransformMeta.getMaxValue()).thenReturn("100");
+ when(transformMockHelper.pipeline.getContainerId())
+ .thenReturn("test-" + System.currentTimeMillis());
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ assertTrue(addSequence.init());
+ assertNotNull(transformMockHelper.iTransformData.counter);
+
+ addSequence.dispose();
+
+ assertNull(transformMockHelper.iTransformData.counter);
+ }
+
+ /** Test init() method with database configuration fails when connection is
missing */
+ @Test
+ void testInitWithDatabaseMissingConnection() {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(false);
+ when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(true);
+ when(transformMockHelper.iTransformMeta.getConnection()).thenReturn("");
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ boolean result = addSequence.init();
+
+ assertFalse(result);
+ }
+
+ /** Test init() method with neither counter nor database fails */
+ @Test
+ void testInitWithNoMethod() {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(false);
+
when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(false);
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ boolean result = addSequence.init();
+
+ assertFalse(result);
+ }
+
+ /** Test addSequence() method with counter */
+ @Test
+ void testAddSequenceMethodWithCounter() throws HopException {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(true);
+
when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(false);
+ when(transformMockHelper.iTransformMeta.getValueName()).thenReturn("id");
+ when(transformMockHelper.iTransformMeta.getStartAt()).thenReturn("1");
+ when(transformMockHelper.iTransformMeta.getIncrementBy()).thenReturn("1");
+ when(transformMockHelper.iTransformMeta.getMaxValue()).thenReturn("100");
+ when(transformMockHelper.pipeline.getContainerId())
+ .thenReturn("test-" + System.currentTimeMillis());
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ assertTrue(addSequence.init());
+
+ // Create input row
+ IRowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("name"));
+ Object[] inputRow = new Object[] {"Alice"};
+
+ // Test addSequence method
+ Object[] result = addSequence.addSequence(inputRowMeta, inputRow);
+
+ assertNotNull(result);
+ assertEquals("Alice", result[0]);
+ assertEquals(1L, result[1]);
+
+ // Test second call
+ result = addSequence.addSequence(inputRowMeta, new Object[] {"Bob"});
+ assertNotNull(result);
+ assertEquals("Bob", result[0]);
+ assertEquals(2L, result[1]);
+
+ addSequence.dispose();
+ }
+
+ /** Test addSequence() method with database */
+ @Test
+ void testAddSequenceMethodWithDatabase() throws HopException {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(false);
+ when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(true);
+ when(transformMockHelper.iTransformMeta.getValueName()).thenReturn("id");
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ // Mock database - NOTE: This doesn't work as expected
+ // The database needs to be set AFTER init() or through a different
mechanism
+ Database db = mock(Database.class);
+ when(db.getNextSequenceValue("public", "test_seq", "id")).thenReturn(100L,
101L);
+ when(addSequence.getData().getDb()).thenReturn(db);
+ transformMockHelper.iTransformData.setDb(db);
+ transformMockHelper.iTransformData.realSchemaName = "public";
+ transformMockHelper.iTransformData.realSequenceName = "test_seq";
+
+ // Create input row
+ IRowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("name"));
+ Object[] inputRow = new Object[] {"Charlie"};
+
+ // Test addSequence method
+ Object[] result = addSequence.addSequence(inputRowMeta, inputRow);
+
+ assertNotNull(result);
+ assertEquals("Charlie", result[0]);
+ assertEquals(100L, result[1]);
+
+ // Test second call
+ result = addSequence.addSequence(inputRowMeta, new Object[] {"Dave"});
+ assertNotNull(result);
+ assertEquals("Dave", result[0]);
+ assertEquals(101L, result[1]);
+
+ addSequence.dispose();
+ }
+
+ /** Test processRow() method with counter-based sequence */
+ @Test
+ void testProcessRowWithCounter() throws Exception {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(true);
+
when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(false);
+ when(transformMockHelper.iTransformMeta.getValueName()).thenReturn("id");
+ when(transformMockHelper.iTransformMeta.getStartAt()).thenReturn("1");
+ when(transformMockHelper.iTransformMeta.getIncrementBy()).thenReturn("1");
+ when(transformMockHelper.iTransformMeta.getMaxValue()).thenReturn("100");
+ when(transformMockHelper.pipeline.getContainerId())
+ .thenReturn("test-processrow-" + System.currentTimeMillis());
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ assertTrue(addSequence.init());
+
+ // Set up input row meta
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("name"));
+ inputRowMeta.addValueMeta(new ValueMetaInteger("age"));
+ addSequence.setInputRowMeta(inputRowMeta);
+
+ // Spy on the transform to mock getRow()
+ addSequence = spy(addSequence);
+ doReturn(new Object[] {"Alice", 30L})
+ .doReturn(new Object[] {"Bob", 25L})
+ .doReturn(new Object[] {"Charlie", 35L})
+ .doReturn(null)
+ .when(addSequence)
+ .getRow();
+
+ // Execute processRow multiple times
+ List<Object[]> result = PipelineTestingUtil.execute(addSequence, 3, false);
+
+ // Verify results
+ assertEquals(3, result.size());
+
+ // First row
+ assertEquals(3, result.size());
+ assertEquals("Alice", result.get(0)[0]);
+ assertEquals(30L, result.get(0)[1]);
+ assertEquals(1L, result.get(0)[2]);
+
+ // Second row
+ assertEquals("Bob", result.get(1)[0]);
+ assertEquals(25L, result.get(1)[1]);
+ assertEquals(2L, result.get(1)[2]);
+
+ // Third row
+ assertEquals("Charlie", result.get(2)[0]);
+ assertEquals(35L, result.get(2)[1]);
+ assertEquals(3L, result.get(2)[2]);
+
+ addSequence.dispose();
+ }
+
+ /** Test processRow() method with database-based sequence */
+ @Test
+ void testProcessRowWithDatabase() throws Exception {
+ when(transformMockHelper.iTransformMeta.isCounterUsed()).thenReturn(false);
+ when(transformMockHelper.iTransformMeta.isDatabaseUsed()).thenReturn(true);
+ when(transformMockHelper.iTransformMeta.getValueName()).thenReturn("id");
+
+ AddSequence addSequence =
+ new AddSequence(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ // Mock database - NOTE: This approach doesn't work with spy()
+ Database db = mock(Database.class);
+ when(db.getNextSequenceValue("public", "my_seq", "id")).thenReturn(100L,
101L, 102L);
+ when(addSequence.getData().getDb()).thenReturn(db);
+ transformMockHelper.iTransformData.setDb(db);
+ transformMockHelper.iTransformData.realSchemaName = "public";
+ transformMockHelper.iTransformData.realSequenceName = "my_seq";
+
+ // Set up input row meta
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("name"));
+ addSequence.setInputRowMeta(inputRowMeta);
+
+ // Spy on the transform to mock getRow()
+ addSequence = spy(addSequence);
+ doReturn(new Object[] {"User1"})
+ .doReturn(new Object[] {"User2"})
+ .doReturn(new Object[] {"User3"})
+ .doReturn(null)
+ .when(addSequence)
+ .getRow();
+
+ // Execute processRow multiple times
+ List<Object[]> result = PipelineTestingUtil.execute(addSequence, 3, false);
+
+ // Verify results
+ assertEquals(3, result.size());
+
+ // First row
+ assertEquals("User1", result.get(0)[0]);
+ assertEquals(100L, result.get(0)[1]);
+
+ // Second row
+ assertEquals("User2", result.get(1)[0]);
+ assertEquals(101L, result.get(1)[1]);
+
+ // Third row
+ assertEquals("User3", result.get(2)[0]);
+ assertEquals(102L, result.get(2)[1]);
+
+ addSequence.dispose();
+ }
+}
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutput.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutput.java
index d56a8eea38..5691df4f64 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutput.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonOutput.java
@@ -101,8 +101,8 @@ public class JsonOutput extends
BaseTransform<JsonOutputMeta, JsonOutputData> {
@Override
public boolean processRow() throws HopException {
-
- Object[] r = getRow(); // This also waits for a row to be finished.
+ // This also waits for a row to be finished.
+ Object[] r = getRow();
if (r == null) {
// only attempt writing to file when the first row is not empty
if (data.isWriteToFile && !first && meta.getSplitOutputAfter() == 0) {
@@ -110,6 +110,8 @@ public class JsonOutput extends
BaseTransform<JsonOutputMeta, JsonOutputData> {
// Let's output the remaining unsafe data
outputRow(prevRow);
writeJsonFile();
+ setOutputDone();
+ return false;
}
// Process the leftover data only when a split file size is defined
@@ -117,7 +119,11 @@ public class JsonOutput extends
BaseTransform<JsonOutputMeta, JsonOutputData> {
if (meta.getSplitOutputAfter() > 0 && !data.jsonItems.isEmpty()) {
serializeJson(data.jsonItems);
writeJsonFile();
+ setOutputDone();
+ return false;
}
+
+ outputRow(prevRow);
setOutputDone();
return false;
}
