This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/master by this push:
new f78910d HOP-2837: add final database transforms
new 14ebe89 Merge pull request #786 from hansva/master
f78910d is described below
commit f78910db4a127257ad3ef730dba960d6186d15b1
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Sat May 1 12:03:28 2021 +0200
HOP-2837: add final database transforms
---
integration-tests/database/0006-delete.hpl | 114 ++++++++++
.../database/0011-combination-lookup-update.hpl | 120 ++++++++++
.../database/0012-dimension-lookup-update.hpl | 152 +++++++++++++
.../database/0013-synchronize-after-merge.hpl | 246 +++++++++++++++++++++
.../database/main-0006-insert-update-delete.hwf | 80 ++++++-
.../main-0011-combination-lookup-update.hwf | 125 +++++++++++
.../database/main-0012-dimension-lookup-update.hwf | 125 +++++++++++
.../database/main-0013-synchronize-after-merge.hwf | 181 +++++++++++++++
integration-tests/database/scripts/script3.sql | 23 ++
integration-tests/database/scripts/script4.sql | 32 +++
10 files changed, 1190 insertions(+), 8 deletions(-)
diff --git a/integration-tests/database/0006-delete.hpl
b/integration-tests/database/0006-delete.hpl
new file mode 100644
index 0000000..89111b9
--- /dev/null
+++ b/integration-tests/database/0006-delete.hpl
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>0006-delete</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/04/29 14:28:18.668</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/04/29 14:28:18.668</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Data grid</from>
+ <to>Delete</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Data grid</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>key</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>value</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>10</item>
+ <item>aa</item>
+ </line>
+ <line>
+ <item>20</item>
+ <item>bb</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Delete</name>
+ <type>Delete</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <connection>unit-test-db</connection>
+ <commit>100</commit>
+ <lookup>
+ <schema>public</schema>
+ <table>testtable</table>
+ <key>
+ <name>key</name>
+ <field>key</field>
+ <condition>=</condition>
+ <name2/>
+ </key>
+ </lookup>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/database/0011-combination-lookup-update.hpl
b/integration-tests/database/0011-combination-lookup-update.hpl
new file mode 100644
index 0000000..69206c6
--- /dev/null
+++ b/integration-tests/database/0011-combination-lookup-update.hpl
@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>0011-combination-lookup-update</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/04/30 11:01:28.333</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/04/30 11:01:28.333</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Data grid</from>
+ <to>Combination lookup/update</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Combination lookup/update</name>
+ <type>CombinationLookup</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <schema>public</schema>
+ <table>testtable</table>
+ <connection>unit-test-db</connection>
+ <commit>100</commit>
+ <cache_size>9999</cache_size>
+ <replace>N</replace>
+ <preloadCache>N</preloadCache>
+ <crc>N</crc>
+ <crcfield>hashcode</crcfield>
+ <fields>
+ <key>
+ <name>value</name>
+ <lookup>value</lookup>
+ </key>
+ <return>
+ <name>key</name>
+ <creation_method>tablemax</creation_method>
+ <use_autoinc>N</use_autoinc>
+ </return>
+ </fields>
+ <sequence/>
+ <last_update_field/>
+ <attributes/>
+ <GUI>
+ <xloc>368</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Data grid</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>value</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>a</item>
+ </line>
+ <line>
+ <item>b</item>
+ </line>
+ <line>
+ <item>c</item>
+ </line>
+ <line>
+ <item>d</item>
+ </line>
+ <line>
+ <item>e</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/database/0012-dimension-lookup-update.hpl
b/integration-tests/database/0012-dimension-lookup-update.hpl
new file mode 100644
index 0000000..ec5b31f
--- /dev/null
+++ b/integration-tests/database/0012-dimension-lookup-update.hpl
@@ -0,0 +1,152 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>0012-dimension-lookup-update</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/04/30 11:01:28.333</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/04/30 11:01:28.333</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Data grid</from>
+ <to>Dimension lookup/update</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Data grid</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>key</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>value</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>a</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>b</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>c</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>d</item>
+ </line>
+ <line>
+ <item>5</item>
+ <item>e</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Dimension lookup/update</name>
+ <type>DimensionLookup</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <schema>public</schema>
+ <table>dimension</table>
+ <connection>unit-test-db</connection>
+ <commit>100</commit>
+ <update>Y</update>
+ <fields>
+ <key>
+ <name>key</name>
+ <lookup>key</lookup>
+ </key>
+ <date>
+ <name/>
+ <from>date_from</from>
+ <to>date_to</to>
+ </date>
+ <field>
+ <name>value</name>
+ <lookup>value</lookup>
+ <update>Insert</update>
+ </field>
+ <return>
+ <name>dimension_id</name>
+ <rename/>
+ <creation_method>tablemax</creation_method>
+ <use_autoinc>N</use_autoinc>
+ <version>version</version>
+ </return>
+ </fields>
+ <sequence/>
+ <min_year>1900</min_year>
+ <max_year>2199</max_year>
+ <cache_size>5000</cache_size>
+ <preload_cache>N</preload_cache>
+ <use_start_date_alternative>N</use_start_date_alternative>
+ <start_date_alternative>none</start_date_alternative>
+ <start_date_field_name/>
+ <useBatch>N</useBatch>
+ <attributes/>
+ <GUI>
+ <xloc>364</xloc>
+ <yloc>94</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/database/0013-synchronize-after-merge.hpl
b/integration-tests/database/0013-synchronize-after-merge.hpl
new file mode 100644
index 0000000..dc8338b
--- /dev/null
+++ b/integration-tests/database/0013-synchronize-after-merge.hpl
@@ -0,0 +1,246 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pipeline>
+ <info>
+ <name>0013-synchronize-after-merge</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/04/30 11:01:28.333</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/04/30 11:01:28.333</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>source</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>new</from>
+ <to>Merge rows (diff)</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Merge rows (diff)</from>
+ <to>Synchronize after merge</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>source</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>key</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>value</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>10</item>
+ <item>aa</item>
+ </line>
+ <line>
+ <item>20</item>
+ <item>bb</item>
+ </line>
+ <line>
+ <item>30</item>
+ <item>cc</item>
+ </line>
+ <line>
+ <item>40</item>
+ <item>dd</item>
+ </line>
+ <line>
+ <item>50</item>
+ <item>ee</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Merge rows (diff)</name>
+ <type>MergeRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <keys>
+ <key>key</key>
+ </keys>
+ <values>
+ <value>value</value>
+ </values>
+ <flag_field>flagfield</flag_field>
+ <reference>source</reference>
+ <compare>new</compare>
+ <compare>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>336</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Synchronize after merge</name>
+ <type>SynchronizeAfterMerge</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <connection>unit-test-db</connection>
+ <commit>100</commit>
+ <tablename_in_field>N</tablename_in_field>
+ <tablename_field/>
+ <use_batch>Y</use_batch>
+ <perform_lookup>N</perform_lookup>
+ <operation_order_field>flagfield</operation_order_field>
+ <order_insert>new</order_insert>
+ <order_update>changed</order_update>
+ <order_delete>deleted</order_delete>
+ <lookup>
+ <schema>public</schema>
+ <table>testtable</table>
+ <key>
+ <name>key</name>
+ <field>key</field>
+ <condition>=</condition>
+ <name2/>
+ </key>
+ <value>
+ <name>key</name>
+ <rename>key</rename>
+ <update>N</update>
+ </value>
+ <value>
+ <name>value</name>
+ <rename>value</rename>
+ <update>Y</update>
+ </value>
+ </lookup>
+ <attributes/>
+ <GUI>
+ <xloc>560</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>new</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>key</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>value</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>10</item>
+ <item>aa</item>
+ </line>
+ <line>
+ <item>30</item>
+ <item>cc</item>
+ </line>
+ <line>
+ <item>40</item>
+ <item>dd</item>
+ </line>
+ <line>
+ <item>50</item>
+ <item>e</item>
+ </line>
+ <line>
+ <item>60</item>
+ <item>ff</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>336</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/database/main-0006-insert-update-delete.hwf
b/integration-tests/database/main-0006-insert-update-delete.hwf
index abb21d9..cbcb7ad 100644
--- a/integration-tests/database/main-0006-insert-update-delete.hwf
+++ b/integration-tests/database/main-0006-insert-update-delete.hwf
@@ -55,7 +55,7 @@
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<parallel>N</parallel>
- <xloc>400</xloc>
+ <xloc>256</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@@ -75,7 +75,7 @@
<add_rows_result>N</add_rows_result>
<clear_result_rows>Y</clear_result_rows>
<parallel>N</parallel>
- <xloc>1088</xloc>
+ <xloc>880</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@@ -104,7 +104,7 @@
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<parallel>N</parallel>
- <xloc>1252</xloc>
+ <xloc>1008</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@@ -125,7 +125,7 @@ where value = 'aa'</custom_sql>
<add_rows_result>N</add_rows_result>
<clear_result_rows>Y</clear_result_rows>
<parallel>N</parallel>
- <xloc>1648</xloc>
+ <xloc>1328</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@@ -154,7 +154,7 @@ where value = 'aa'</custom_sql>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<parallel>N</parallel>
- <xloc>658</xloc>
+ <xloc>464</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@@ -183,7 +183,7 @@ where value = 'aa'</custom_sql>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<parallel>N</parallel>
- <xloc>912</xloc>
+ <xloc>720</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@@ -199,7 +199,7 @@ where value = 'aa'</custom_sql>
<sendOneStatement>F</sendOneStatement>
<connection>unit-test-db</connection>
<parallel>N</parallel>
- <xloc>262</xloc>
+ <xloc>160</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@@ -228,7 +228,57 @@ where value = 'aa'</custom_sql>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<parallel>N</parallel>
- <xloc>1450</xloc>
+ <xloc>1168</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0006-delete</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0006-delete.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <xloc>1440</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>check deleted</name>
+ <description/>
+ <type>EVAL_TABLE_CONTENT</type>
+ <attributes/>
+ <connection>unit-test-db</connection>
+ <schemaname/>
+ <tablename/>
+ <success_condition>rows_count_equal</success_condition>
+ <limit>0</limit>
+ <is_custom_sql>Y</is_custom_sql>
+ <is_usevars>N</is_usevars>
+ <custom_sql>select * from public.testtable
+where key = '10'</custom_sql>
+ <add_rows_result>N</add_rows_result>
+ <clear_result_rows>Y</clear_result_rows>
+ <parallel>N</parallel>
+ <xloc>1568</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@@ -290,6 +340,20 @@ where value = 'aa'</custom_sql>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
+ <hop>
+ <from>check new value</from>
+ <to>0006-delete</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0006-delete</from>
+ <to>check deleted</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
</hops>
<notepads>
</notepads>
diff --git a/integration-tests/database/main-0011-combination-lookup-update.hwf
b/integration-tests/database/main-0011-combination-lookup-update.hwf
new file mode 100644
index 0000000..9c23f31
--- /dev/null
+++ b/integration-tests/database/main-0011-combination-lookup-update.hwf
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<workflow>
+ <name>main-0011-combination-lookup-update</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2021/04/29 11:51:40.288</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/04/29 11:51:40.288</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>64</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>SQL</name>
+ <description/>
+ <type>SQL</type>
+ <attributes/>
+ <sql/>
+ <useVariableSubstitution>F</useVariableSubstitution>
+ <sqlfromfile>T</sqlfromfile>
+ <sqlfilename>${PROJECT_HOME}/scripts/script3.sql</sqlfilename>
+ <sendOneStatement>F</sendOneStatement>
+ <connection>unit-test-db</connection>
+ <parallel>N</parallel>
+ <xloc>208</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0011-combination-lookup-update.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0011-combination-lookup-update.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <xloc>368</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Evaluate rows number in a table</name>
+ <description/>
+ <type>EVAL_TABLE_CONTENT</type>
+ <attributes/>
+ <connection>unit-test-db</connection>
+ <schemaname>public</schemaname>
+ <tablename>testtable</tablename>
+ <success_condition>rows_count_equal</success_condition>
+ <limit>5</limit>
+ <is_custom_sql>N</is_custom_sql>
+ <is_usevars>N</is_usevars>
+ <custom_sql/>
+ <add_rows_result>N</add_rows_result>
+ <clear_result_rows>Y</clear_result_rows>
+ <parallel>N</parallel>
+ <xloc>640</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>SQL</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>SQL</from>
+ <to>0011-combination-lookup-update.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0011-combination-lookup-update.hpl</from>
+ <to>Evaluate rows number in a table</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/database/main-0012-dimension-lookup-update.hwf
b/integration-tests/database/main-0012-dimension-lookup-update.hwf
new file mode 100644
index 0000000..92eddc7
--- /dev/null
+++ b/integration-tests/database/main-0012-dimension-lookup-update.hwf
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<workflow>
+ <name>main-0012-dimension-lookup-update</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2021/04/29 11:51:40.288</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/04/29 11:51:40.288</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>64</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>SQL</name>
+ <description/>
+ <type>SQL</type>
+ <attributes/>
+ <sql/>
+ <useVariableSubstitution>F</useVariableSubstitution>
+ <sqlfromfile>T</sqlfromfile>
+ <sqlfilename>${PROJECT_HOME}/scripts/script4.sql</sqlfilename>
+ <sendOneStatement>F</sendOneStatement>
+ <connection>unit-test-db</connection>
+ <parallel>N</parallel>
+ <xloc>208</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0012-dimension-lookup-update.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0012-dimension-lookup-update.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <xloc>368</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Evaluate rows number in a table</name>
+ <description/>
+ <type>EVAL_TABLE_CONTENT</type>
+ <attributes/>
+ <connection>unit-test-db</connection>
+ <schemaname>public</schemaname>
+ <tablename>dimension</tablename>
+ <success_condition>rows_count_equal</success_condition>
+ <limit>6</limit>
+ <is_custom_sql>N</is_custom_sql>
+ <is_usevars>N</is_usevars>
+ <custom_sql/>
+ <add_rows_result>N</add_rows_result>
+ <clear_result_rows>Y</clear_result_rows>
+ <parallel>N</parallel>
+ <xloc>640</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>SQL</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>SQL</from>
+ <to>0012-dimension-lookup-update.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0012-dimension-lookup-update.hpl</from>
+ <to>Evaluate rows number in a table</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/database/main-0013-synchronize-after-merge.hwf
b/integration-tests/database/main-0013-synchronize-after-merge.hwf
new file mode 100644
index 0000000..84113da
--- /dev/null
+++ b/integration-tests/database/main-0013-synchronize-after-merge.hwf
@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<workflow>
+ <name>main-0013-synchronize-after-merge</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2021/04/29 11:51:40.288</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/04/29 11:51:40.288</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>64</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>SQL</name>
+ <description/>
+ <type>SQL</type>
+ <attributes/>
+ <sql/>
+ <useVariableSubstitution>F</useVariableSubstitution>
+ <sqlfromfile>T</sqlfromfile>
+ <sqlfilename>${PROJECT_HOME}/scripts/script2.sql</sqlfilename>
+ <sendOneStatement>F</sendOneStatement>
+ <connection>unit-test-db</connection>
+ <parallel>N</parallel>
+ <xloc>230</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0013-synchronize-after-merge.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0013-synchronize-after-merge.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <xloc>396</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>count rows</name>
+ <description/>
+ <type>EVAL_TABLE_CONTENT</type>
+ <attributes/>
+ <connection>unit-test-db</connection>
+ <schemaname>public</schemaname>
+ <tablename>testtable</tablename>
+ <success_condition>rows_count_equal</success_condition>
+ <limit>5</limit>
+ <is_custom_sql>N</is_custom_sql>
+ <is_usevars>N</is_usevars>
+ <custom_sql/>
+ <add_rows_result>N</add_rows_result>
+ <clear_result_rows>Y</clear_result_rows>
+ <parallel>N</parallel>
+ <xloc>562</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>check changed value</name>
+ <description/>
+ <type>EVAL_TABLE_CONTENT</type>
+ <attributes/>
+ <connection>unit-test-db</connection>
+ <schemaname>public</schemaname>
+ <tablename>testtable</tablename>
+ <success_condition>rows_count_equal</success_condition>
+ <limit>1</limit>
+ <is_custom_sql>Y</is_custom_sql>
+ <is_usevars>N</is_usevars>
+ <custom_sql>select 1 from public.testtable
+where value = 'e'</custom_sql>
+ <add_rows_result>N</add_rows_result>
+ <clear_result_rows>Y</clear_result_rows>
+ <parallel>N</parallel>
+ <xloc>728</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>check deleted row</name>
+ <description/>
+ <type>EVAL_TABLE_CONTENT</type>
+ <attributes/>
+ <connection>unit-test-db</connection>
+ <schemaname>public</schemaname>
+ <tablename>testtable</tablename>
+ <success_condition>rows_count_equal</success_condition>
+ <limit>0</limit>
+ <is_custom_sql>Y</is_custom_sql>
+ <is_usevars>N</is_usevars>
+ <custom_sql>select 1 from public.testtable
+where value = 'bb'</custom_sql>
+ <add_rows_result>N</add_rows_result>
+ <clear_result_rows>Y</clear_result_rows>
+ <parallel>N</parallel>
+ <xloc>894</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>SQL</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>SQL</from>
+ <to>0013-synchronize-after-merge.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0013-synchronize-after-merge.hpl</from>
+ <to>count rows</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>count rows</from>
+ <to>check changed value</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>check changed value</from>
+ <to>check deleted row</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/database/scripts/script3.sql
b/integration-tests/database/scripts/script3.sql
new file mode 100644
index 0000000..d8d3581
--- /dev/null
+++ b/integration-tests/database/scripts/script3.sql
@@ -0,0 +1,23 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+DROP TABLE IF EXISTS public.testtable;
+
+CREATE TABLE public.testtable (
+ "key" integer NULL,
+ value varchar NULL
+);
diff --git a/integration-tests/database/scripts/script4.sql
b/integration-tests/database/scripts/script4.sql
new file mode 100644
index 0000000..39565d4
--- /dev/null
+++ b/integration-tests/database/scripts/script4.sql
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+DROP TABLE IF EXISTS public.dimension;
+
+CREATE TABLE public.dimension
+(
+ dimension_id BIGSERIAL
+, version INTEGER
+, date_from TIMESTAMP
+, date_to TIMESTAMP
+, "key" TEXT
+, "value" TEXT
+)
+;CREATE INDEX idx_dimension_lookup ON "public".dimension("key")
+;
+CREATE INDEX idx_dimension_tk ON "public".dimension(dimension_id)
+;
