This is an automated email from the ASF dual-hosted git repository.
mcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new a7ce99e73a issue #1936 (XML cleanup Stream Lookup) (#6874)
a7ce99e73a is described below
commit a7ce99e73af1e43a7484537c9ef6446a57aef08c
Author: Matt Casters <[email protected]>
AuthorDate: Fri Mar 27 13:00:22 2026 +0100
issue #1936 (XML cleanup Stream Lookup) (#6874)
---
.../mdi/0048-stream-lookup-template.hpl | 68 +++
.../mdi/0048-stream-lookup-validate.hpl | 435 ++++++++++++++++++
integration-tests/mdi/0048-stream-lookup.hpl | 446 +++++++++++++++++++
.../mdi/datasets/golden-stream-lookup-keys.csv | 3 +
.../mdi/datasets/golden-stream-lookup-values.csv | 4 +
.../mdi/datasets/golden-stream-lookup.csv | 2 +
integration-tests/mdi/files/stream-lookup.xml | 52 +++
integration-tests/mdi/main-0048-stream-lookup.hwf | 113 +++++
.../dataset/golden-stream-lookup-keys.json | 24 +
.../dataset/golden-stream-lookup-values.json | 40 ++
.../mdi/metadata/dataset/golden-stream-lookup.json | 48 ++
.../0048-stream-lookup-validate UNIT.json | 90 ++++
...PipelineTest.java => MatchKeyPipelineTest.java} | 2 +-
.../apache/hop/beam/util/BeamPipelineMetaUtil.java | 24 +-
.../transforms/streamlookup/StreamLookup.java | 163 +++----
.../streamlookup/StreamLookupDialog.java | 105 ++---
.../transforms/streamlookup/StreamLookupMeta.java | 491 +++++++--------------
.../messages/messages_en_US.properties | 4 +
.../streamlookup/StreamLookupMetaTest.java | 160 ++-----
.../transforms/streamlookup/StreamLookupTest.java | 259 -----------
.../src/test/resources/stream-lookup.xml | 52 +++
21 files changed, 1711 insertions(+), 874 deletions(-)
diff --git a/integration-tests/mdi/0048-stream-lookup-template.hpl
b/integration-tests/mdi/0048-stream-lookup-template.hpl
new file mode 100644
index 0000000000..b9d0248c41
--- /dev/null
+++ b/integration-tests/mdi/0048-stream-lookup-template.hpl
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0048-stream-lookup-template</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/03/25 18:24:50.466</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/25 18:24:50.466</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ </order>
+ <transform>
+ <name>Stream lookup</name>
+ <type>StreamLookup</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <input_sorted>N</input_sorted>
+ <integer_pair>N</integer_pair>
+ <lookup>
+</lookup>
+ <preserve_memory>Y</preserve_memory>
+ <sorted_list>N</sorted_list>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/mdi/0048-stream-lookup-validate.hpl
b/integration-tests/mdi/0048-stream-lookup-validate.hpl
new file mode 100644
index 0000000000..345c54bfda
--- /dev/null
+++ b/integration-tests/mdi/0048-stream-lookup-validate.hpl
@@ -0,0 +1,435 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0048-stream-lookup-validate</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/03/25 18:30:51.794</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/25 18:30:51.794</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>/tmp/stream-lookup.hpl</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>/tmp/stream-lookup.hpl keys</from>
+ <to>validate keys</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>/tmp/stream-lookup.hpl values</from>
+ <to>validate values</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>/tmp/stream-lookup.hpl</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <XmlField/>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <extensionFieldName/>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>from</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>from</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>input_sorted</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>input_sorted</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>integer_pair</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>integer_pair</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>preserve_memory</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>preserve_memory</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>sorted_list</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>sorted_list</xPath>
+ </field>
+ </fields>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/stream-lookup.hpl</name>
+ </file>
+ <hiddenFieldName/>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
+ <limit>0</limit>
+ <loopxpath>/pipeline/transform</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
+ <readurl>N</readurl>
+ <rootUriNameFieldName/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>/tmp/stream-lookup.hpl keys</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <XmlField/>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <extensionFieldName/>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>field</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>field</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>name</xPath>
+ </field>
+ </fields>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/stream-lookup.hpl</name>
+ </file>
+ <hiddenFieldName/>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
+ <limit>0</limit>
+ <loopxpath>/pipeline/transform/lookup/key</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
+ <readurl>N</readurl>
+ <rootUriNameFieldName/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate keys</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>/tmp/stream-lookup.hpl values</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <XmlField/>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <extensionFieldName/>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>default</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>default</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>name</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>rename</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>rename</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>type</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>type</xPath>
+ </field>
+ </fields>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/stream-lookup.hpl</name>
+ </file>
+ <hiddenFieldName/>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
+ <limit>0</limit>
+ <loopxpath>/pipeline/transform/lookup/value</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
+ <readurl>N</readurl>
+ <rootUriNameFieldName/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate values</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/mdi/0048-stream-lookup.hpl
b/integration-tests/mdi/0048-stream-lookup.hpl
new file mode 100644
index 0000000000..3029a4dc84
--- /dev/null
+++ b/integration-tests/mdi/0048-stream-lookup.hpl
@@ -0,0 +1,446 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0048-stream-lookup</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/03/25 18:25:54.053</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/25 18:25:54.053</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>files/stream-lookup.xml</from>
+ <to>0048-stream-lookup.hpl</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>files/stream-lookup.xml keys</from>
+ <to>0048-stream-lookup.hpl</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>files/stream-lookup.xml values</from>
+ <to>0048-stream-lookup.hpl</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>0048-stream-lookup.hpl</name>
+ <type>MetaInject</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
+ <create_parent_folder>Y</create_parent_folder>
+ <filename>${PROJECT_HOME}/0048-stream-lookup-template.hpl</filename>
+ <mappings>
+ <mapping>
+ <source_field>from</source_field>
+ <source_transform>files/stream-lookup.xml</source_transform>
+ <target_attribute_key>SOURCE_TRANSFORM</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>input_sorted</source_field>
+ <source_transform>files/stream-lookup.xml</source_transform>
+ <target_attribute_key>INPUT_SORTED</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>preserve_memory</source_field>
+ <source_transform>files/stream-lookup.xml</source_transform>
+ <target_attribute_key>PRESERVE_MEMORY</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>sorted_list</source_field>
+ <source_transform>files/stream-lookup.xml</source_transform>
+ <target_attribute_key>SORTED_LIST</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>integer_pair</source_field>
+ <source_transform>files/stream-lookup.xml</source_transform>
+ <target_attribute_key>INTEGER_PAIR</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>name</source_field>
+ <source_transform>files/stream-lookup.xml keys</source_transform>
+ <target_attribute_key>KEY_STREAM</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>field</source_field>
+ <source_transform>files/stream-lookup.xml keys</source_transform>
+ <target_attribute_key>KEY_LOOKUP</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>name</source_field>
+ <source_transform>files/stream-lookup.xml values</source_transform>
+ <target_attribute_key>RETRIEVE_VALUE</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>rename</source_field>
+ <source_transform>files/stream-lookup.xml values</source_transform>
+ <target_attribute_key>RETRIEVE_VALUE_NAME</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>default</source_field>
+ <source_transform>files/stream-lookup.xml values</source_transform>
+ <target_attribute_key>RETRIEVE_VALUE_DEFAULT</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>type</source_field>
+ <source_transform>files/stream-lookup.xml values</source_transform>
+ <target_attribute_key>RETRIEVE_DEFAULT_TYPE</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>Stream lookup</target_transform_name>
+ </mapping>
+ </mappings>
+ <no_execution>Y</no_execution>
+ <run_configuration>local</run_configuration>
+ <source_output_fields>
+</source_output_fields>
+ <target_file>${java.io.tmpdir}/stream-lookup.hpl</target_file>
+ <attributes/>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>files/stream-lookup.xml</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>from</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>from</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>input_sorted</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>input_sorted</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>preserve_memory</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>preserve_memory</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>sorted_list</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>sorted_list</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>integer_pair</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>integer_pair</xPath>
+ </field>
+ </fields>
+ <file>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ <name>${PROJECT_HOME}/files/stream-lookup.xml</name>
+ </file>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <limit>0</limit>
+ <loopxpath>/transform</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <readurl>N</readurl>
+ <rownum>N</rownum>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>files/stream-lookup.xml keys</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>name</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>field</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>field</xPath>
+ </field>
+ </fields>
+ <file>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ <name>${PROJECT_HOME}/files/stream-lookup.xml</name>
+ </file>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <limit>0</limit>
+ <loopxpath>/transform/lookup/key</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <readurl>N</readurl>
+ <rownum>N</rownum>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>files/stream-lookup.xml values</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>name</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>name</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>rename</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>rename</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>default</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>default</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>type</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>type</xPath>
+ </field>
+ </fields>
+ <file>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ <name>${PROJECT_HOME}/files/stream-lookup.xml</name>
+ </file>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <limit>0</limit>
+ <loopxpath>/transform/lookup/value</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <readurl>N</readurl>
+ <rownum>N</rownum>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>192</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/mdi/datasets/golden-stream-lookup-keys.csv
b/integration-tests/mdi/datasets/golden-stream-lookup-keys.csv
new file mode 100644
index 0000000000..dbd48d343f
--- /dev/null
+++ b/integration-tests/mdi/datasets/golden-stream-lookup-keys.csv
@@ -0,0 +1,3 @@
+name,field
+keyStream1,keyLookup1
+keyStream2,keyLookup2
diff --git a/integration-tests/mdi/datasets/golden-stream-lookup-values.csv
b/integration-tests/mdi/datasets/golden-stream-lookup-values.csv
new file mode 100644
index 0000000000..bd9a572a9c
--- /dev/null
+++ b/integration-tests/mdi/datasets/golden-stream-lookup-values.csv
@@ -0,0 +1,4 @@
+name,rename,default,type
+returnValue1,value1,,String
+returnValue2,value2,123,Integer
+returnValue3,value3,true,Boolean
diff --git a/integration-tests/mdi/datasets/golden-stream-lookup.csv
b/integration-tests/mdi/datasets/golden-stream-lookup.csv
new file mode 100644
index 0000000000..7cfc04651f
--- /dev/null
+++ b/integration-tests/mdi/datasets/golden-stream-lookup.csv
@@ -0,0 +1,2 @@
+from,input_sorted,preserve_memory,sorted_list,integer_pair
+Countries,Y,Y,Y,Y
diff --git a/integration-tests/mdi/files/stream-lookup.xml
b/integration-tests/mdi/files/stream-lookup.xml
new file mode 100644
index 0000000000..ee2640a3c0
--- /dev/null
+++ b/integration-tests/mdi/files/stream-lookup.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<transform>
+ <from>Countries</from>
+ <input_sorted>Y</input_sorted>
+ <preserve_memory>Y</preserve_memory>
+ <sorted_list>Y</sorted_list>
+ <integer_pair>Y</integer_pair>
+ <lookup>
+ <key>
+ <name>keyStream1</name>
+ <field>keyLookup1</field>
+ </key>
+ <key>
+ <name>keyStream2</name>
+ <field>keyLookup2</field>
+ </key>
+ <value>
+ <name>returnValue1</name>
+ <rename>value1</rename>
+ <default/>
+ <type>String</type>
+ </value>
+ <value>
+ <name>returnValue2</name>
+ <rename>value2</rename>
+ <default>123</default>
+ <type>Integer</type>
+ </value>
+ <value>
+ <name>returnValue3</name>
+ <rename>value3</rename>
+ <default>true</default>
+ <type>Boolean</type>
+ </value>
+ </lookup>
+</transform>
diff --git a/integration-tests/mdi/main-0048-stream-lookup.hwf
b/integration-tests/mdi/main-0048-stream-lookup.hwf
new file mode 100644
index 0000000000..a5651e16b7
--- /dev/null
+++ b/integration-tests/mdi/main-0048-stream-lookup.hwf
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0048-stream-lookup</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/03/25 18:24:13.232</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/25 18:24:13.232</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>80</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0048-stream-lookup</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0048-stream-lookup.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>224</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0048-stream-lookup-validate UNIT</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0048-stream-lookup-validate UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>416</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0048-stream-lookup</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0048-stream-lookup</from>
+ <to>0048-stream-lookup-validate UNIT</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/mdi/metadata/dataset/golden-stream-lookup-keys.json
b/integration-tests/mdi/metadata/dataset/golden-stream-lookup-keys.json
new file mode 100644
index 0000000000..052242722e
--- /dev/null
+++ b/integration-tests/mdi/metadata/dataset/golden-stream-lookup-keys.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-stream-lookup-keys.csv",
+ "name": "golden-stream-lookup-keys",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "field",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/mdi/metadata/dataset/golden-stream-lookup-values.json
b/integration-tests/mdi/metadata/dataset/golden-stream-lookup-values.json
new file mode 100644
index 0000000000..dff160dab8
--- /dev/null
+++ b/integration-tests/mdi/metadata/dataset/golden-stream-lookup-values.json
@@ -0,0 +1,40 @@
+{
+ "base_filename": "golden-stream-lookup-values.csv",
+ "name": "golden-stream-lookup-values",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "rename",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "default",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "type",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git a/integration-tests/mdi/metadata/dataset/golden-stream-lookup.json
b/integration-tests/mdi/metadata/dataset/golden-stream-lookup.json
new file mode 100644
index 0000000000..56ece7d7fb
--- /dev/null
+++ b/integration-tests/mdi/metadata/dataset/golden-stream-lookup.json
@@ -0,0 +1,48 @@
+{
+ "base_filename": "golden-stream-lookup.csv",
+ "name": "golden-stream-lookup",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "from",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "input_sorted",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "preserve_memory",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "sorted_list",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "integer_pair",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/mdi/metadata/unit-test/0048-stream-lookup-validate
UNIT.json
b/integration-tests/mdi/metadata/unit-test/0048-stream-lookup-validate UNIT.json
new file mode 100644
index 0000000000..3c48f28b78
--- /dev/null
+++ b/integration-tests/mdi/metadata/unit-test/0048-stream-lookup-validate
UNIT.json
@@ -0,0 +1,90 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "from",
+ "data_set_field": "from"
+ },
+ {
+ "transform_field": "input_sorted",
+ "data_set_field": "input_sorted"
+ },
+ {
+ "transform_field": "integer_pair",
+ "data_set_field": "integer_pair"
+ },
+ {
+ "transform_field": "preserve_memory",
+ "data_set_field": "preserve_memory"
+ },
+ {
+ "transform_field": "sorted_list",
+ "data_set_field": "sorted_list"
+ }
+ ],
+ "field_order": [
+ "from",
+ "input_sorted",
+ "preserve_memory",
+ "sorted_list",
+ "integer_pair"
+ ],
+ "data_set_name": "golden-stream-lookup",
+ "transform_name": "validate"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "field",
+ "data_set_field": "field"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ }
+ ],
+ "field_order": [
+ "name"
+ ],
+ "data_set_name": "golden-stream-lookup-keys",
+ "transform_name": "validate keys"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "default",
+ "data_set_field": "default"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ },
+ {
+ "transform_field": "rename",
+ "data_set_field": "rename"
+ },
+ {
+ "transform_field": "type",
+ "data_set_field": "type"
+ }
+ ],
+ "field_order": [
+ "name"
+ ],
+ "data_set_name": "golden-stream-lookup-values",
+ "transform_name": "validate values"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0048-stream-lookup-validate UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0048-stream-lookup-validate.hpl"
+}
\ No newline at end of file
diff --git
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/transform/StreamLookupPipelineTest.java
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/transform/MatchKeyPipelineTest.java
similarity index 95%
rename from
plugins/engines/beam/src/test/java/org/apache/hop/beam/transform/StreamLookupPipelineTest.java
rename to
plugins/engines/beam/src/test/java/org/apache/hop/beam/transform/MatchKeyPipelineTest.java
index b7671e1bfa..3b2a7917f5 100644
---
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/transform/StreamLookupPipelineTest.java
+++
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/transform/MatchKeyPipelineTest.java
@@ -21,7 +21,7 @@ import org.apache.hop.beam.util.BeamPipelineMetaUtil;
import org.apache.hop.pipeline.PipelineMeta;
import org.junit.jupiter.api.Test;
-class StreamLookupPipelineTest extends PipelineTestBase {
+class MatchKeyPipelineTest extends PipelineTestBase {
@Test
void testStreamLookupPipeline() throws Exception {
diff --git
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/util/BeamPipelineMetaUtil.java
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/util/BeamPipelineMetaUtil.java
index d74956e380..a2d9f3d2ee 100644
---
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/util/BeamPipelineMetaUtil.java
+++
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/util/BeamPipelineMetaUtil.java
@@ -343,8 +343,6 @@ public class BeamPipelineMetaUtil {
beamInputTransformMeta.setTransformPluginId(BeamConst.STRING_BEAM_INPUT_PLUGIN_ID);
pipelineMeta.addTransform(beamInputTransformMeta);
- TransformMeta lookupBeamInputTransformMeta = beamInputTransformMeta;
-
// Add a Memory Group By transform which will
MemoryGroupByMeta memoryGroupByMeta = new MemoryGroupByMeta();
memoryGroupByMeta.setGroups(List.of(new GGroup("stateCode")));
@@ -355,23 +353,27 @@ public class BeamPipelineMetaUtil {
TransformMeta memoryGroupByTransformMeta = new
TransformMeta("rowsPerState", memoryGroupByMeta);
pipelineMeta.addTransform(memoryGroupByTransformMeta);
pipelineMeta.addPipelineHop(
- new PipelineHopMeta(lookupBeamInputTransformMeta,
memoryGroupByTransformMeta));
+ new PipelineHopMeta(beamInputTransformMeta,
memoryGroupByTransformMeta));
// Add a Stream Lookup transform ...
//
StreamLookupMeta streamLookupMeta = new StreamLookupMeta();
- streamLookupMeta.allocate(1, 1);
- streamLookupMeta.getKeystream()[0] = "stateCode";
- streamLookupMeta.getKeylookup()[0] = "stateCode";
- streamLookupMeta.getValue()[0] = "rowsPerState";
- streamLookupMeta.getValueName()[0] = "nrPerState";
- streamLookupMeta.getValueDefault()[0] = null;
- streamLookupMeta.getValueDefaultType()[0] = IValueMeta.TYPE_INTEGER;
+ StreamLookupMeta.MatchKey matchKey = new StreamLookupMeta.MatchKey();
+ matchKey.setKeyStream("stateCode");
+ matchKey.setKeyLookup("stateCode");
+ streamLookupMeta.getLookup().getMatchKeys().add(matchKey);
+
+ StreamLookupMeta.ReturnValue returnValue = new
StreamLookupMeta.ReturnValue();
+ returnValue.setValue("rowsPerState");
+ returnValue.setValueName("nrPerState");
+ returnValue.setValueDefaultType(IValueMeta.TYPE_INTEGER);
+ streamLookupMeta.getLookup().getReturnValues().add(returnValue);
+
streamLookupMeta.setMemoryPreservationActive(false);
streamLookupMeta
.getTransformIOMeta()
.getInfoStreams()
- .get(0)
+ .getFirst()
.setTransformMeta(memoryGroupByTransformMeta); // Read from Mem.GroupBy
TransformMeta streamLookupTransformMeta = new TransformMeta("Stream
Lookup", streamLookupMeta);
pipelineMeta.addTransform(streamLookupTransformMeta);
diff --git
a/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookup.java
b/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookup.java
index 10ce69a7fb..b0c25b3296 100644
---
a/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookup.java
+++
b/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookup.java
@@ -58,15 +58,14 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
}
private void handleNullIf() {
- data.nullIf = new Object[meta.getValue().length];
-
- for (int i = 0; i < meta.getValue().length; i++) {
- if (meta.getValueDefaultType()[i] < 0) {
-
- meta.getValueDefaultType()[i] = IValueMeta.TYPE_STRING;
+ data.nullIf = new Object[meta.getLookup().getReturnValues().size()];
+ for (int i = 0; i < data.nullIf.length; i++) {
+ StreamLookupMeta.ReturnValue returnValue =
meta.getLookup().getReturnValues().get(i);
+ if (returnValue.getValueDefaultType() < 0) {
+ returnValue.setValueDefaultType(IValueMeta.TYPE_STRING);
}
data.nullIf[i] = null;
- switch (meta.getValueDefaultType()[i]) {
+ switch (returnValue.getValueDefaultType()) {
case IValueMeta.TYPE_STRING:
if (Utils.isEmpty(data.valueDefault[i])) {
data.nullIf[i] = null;
@@ -116,7 +115,7 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
throw new RuntimeException(
BaseMessages.getString(PKG,
"StreamLookup.Exception.ConversionNotImplemented")
+ " "
- +
ValueMetaFactory.getValueMetaName(meta.getValueDefaultType()[i]));
+ +
ValueMetaFactory.getValueMetaName(returnValue.getValueDefaultType()));
} else {
// no default value given: just set it to null
data.nullIf[i] = null;
@@ -127,7 +126,7 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
}
private boolean readLookupValues() throws HopException {
- data.infoStream = meta.getTransformIOMeta().getInfoStreams().get(0);
+ data.infoStream = meta.getTransformIOMeta().getInfoStreams().getFirst();
if (data.infoStream.getTransformMeta() == null) {
logError(BaseMessages.getString(PKG,
"StreamLookup.Log.NoLookupTransformSpecified"));
return false;
@@ -139,8 +138,8 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
+ "]");
}
- int[] keyNrs = new int[meta.getKeylookup().length];
- int[] valueNrs = new int[meta.getValue().length];
+ int[] keyNrs = new int[meta.getLookup().getMatchKeys().size()];
+ int[] valueNrs = new int[meta.getLookup().getReturnValues().size()];
boolean firstRun = true;
// Which row set do we read from?
@@ -163,12 +162,13 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
IRowMeta cacheValueMeta = new RowMeta();
// Look up the keys in the source rows
- for (int i = 0; i < meta.getKeylookup().length; i++) {
- keyNrs[i] = rowSet.getRowMeta().indexOfValue(meta.getKeylookup()[i]);
+ for (int i = 0; i < meta.getLookup().getMatchKeys().size(); i++) {
+ StreamLookupMeta.MatchKey matchKey =
meta.getLookup().getMatchKeys().get(i);
+ keyNrs[i] =
rowSet.getRowMeta().indexOfValue(matchKey.getKeyLookup());
if (keyNrs[i] < 0) {
throw new HopTransformException(
BaseMessages.getString(
- PKG, "StreamLookup.Exception.UnableToFindField",
meta.getKeylookup()[i]));
+ PKG, "StreamLookup.Exception.UnableToFindField",
matchKey.getKeyLookup()));
}
cacheKeyMeta.addValueMeta(rowSet.getRowMeta().getValueMeta(keyNrs[i]));
}
@@ -182,12 +182,13 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
cacheKeyMeta.getValueMeta(i).setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
}
- for (int v = 0; v < meta.getValue().length; v++) {
- valueNrs[v] = rowSet.getRowMeta().indexOfValue(meta.getValue()[v]);
+ for (int v = 0; v < meta.getLookup().getReturnValues().size(); v++) {
+ StreamLookupMeta.ReturnValue returnValue =
meta.getLookup().getReturnValues().get(v);
+ valueNrs[v] =
rowSet.getRowMeta().indexOfValue(returnValue.getValue());
if (valueNrs[v] < 0) {
throw new HopTransformException(
BaseMessages.getString(
- PKG, "StreamLookup.Exception.UnableToFindField",
meta.getValue()[v]));
+ PKG, "StreamLookup.Exception.UnableToFindField",
returnValue.getValue()));
}
cacheValueMeta.addValueMeta(rowSet.getRowMeta().getValueMeta(valueNrs[v]));
}
@@ -274,7 +275,7 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
if (data.hasLookupRows) {
try {
- if (meta.getKeystream().length > 0) {
+ if (!meta.getLookup().getMatchKeys().isEmpty()) {
add = getFromCache(data.cacheKeyMeta, lu);
} else {
// Just take the first element in the hashtable...
@@ -379,7 +380,6 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
@Override
public boolean processRow() throws HopException {
-
if (data.readLookupValues) {
data.readLookupValues = false;
@@ -391,15 +391,7 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
return false;
}
- // Calculate the default values by doing variable replacement.
- //
- data.valueDefault = new String[meta.getValueDefault().length];
- for (int i = 0; i < data.valueDefault.length; i++) {
- // We want to do this variable replacement only once since it's a
relatively slow operation.
- //
- data.valueDefault[i] = resolve(meta.getValueDefault()[i]);
- }
-
+ calculateDefaultValues();
return true;
}
@@ -418,66 +410,17 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
if (first) {
first = false;
-
- // read the lookup values!
- data.keynrs = new int[meta.getKeystream().length];
- data.lookupMeta = new RowMeta();
- data.convertKeysToNative = new boolean[meta.getKeystream().length];
-
- for (int i = 0; i < meta.getKeystream().length; i++) {
- // Find the keynr in the row (only once)
- data.keynrs[i] =
getInputRowMeta().indexOfValue(meta.getKeystream()[i]);
- if (data.keynrs[i] < 0) {
- throw new HopTransformException(
- BaseMessages.getString(
- PKG,
- "StreamLookup.Log.FieldNotFound",
- meta.getKeystream()[i],
- "" + getInputRowMeta().getString(r)));
- } else {
- if (isDetailed()) {
- logDetailed(
- BaseMessages.getString(
- PKG,
- "StreamLookup.Log.FieldInfo",
- meta.getKeystream()[i],
- "" + data.keynrs[i]));
- }
- }
-
-
data.lookupMeta.addValueMeta(getInputRowMeta().getValueMeta(data.keynrs[i]).clone());
-
- // If we have binary storage data coming in, we convert it to normal
data storage.
- // The storage in the lookup data store is also normal data storage.
TODO: enforce normal
- // data storage??
- //
- data.convertKeysToNative[i] =
-
getInputRowMeta().getValueMeta(data.keynrs[i]).isStorageBinaryString();
- }
-
- data.outputRowMeta = getInputRowMeta().clone();
- meta.getFields(
- data.outputRowMeta,
- getTransformName(),
- new IRowMeta[] {data.infoMeta},
- null,
- this,
- metadataProvider);
-
- // Handle the NULL values (not found...)
- handleNullIf();
+ processRowFirstCall(r);
}
Object[] outputRow =
lookupValues(getInputRowMeta(), r); // Do the actual lookup in the
hastable.
if (outputRow == null) {
setOutputDone(); // signal end to receiver(s)
-
return false;
}
putRow(data.outputRowMeta, outputRow); // copy row to output rowset(s)
-
if (checkFeedback(getLinesRead()) && isBasic()) {
logBasic(BaseMessages.getString(PKG, "StreamLookup.Log.LineNumber") +
getLinesRead());
}
@@ -485,15 +428,73 @@ public class StreamLookup extends
BaseTransform<StreamLookupMeta, StreamLookupDa
return true;
}
+ private void processRowFirstCall(Object[] r) throws HopTransformException,
HopValueException {
+ // read the lookup values!
+ data.keynrs = new int[meta.getLookup().getMatchKeys().size()];
+ data.lookupMeta = new RowMeta();
+ data.convertKeysToNative = new
boolean[meta.getLookup().getMatchKeys().size()];
+
+ for (int i = 0; i < meta.getLookup().getMatchKeys().size(); i++) {
+ StreamLookupMeta.MatchKey key = meta.getLookup().getMatchKeys().get(i);
+
+ // Find the keynr in the row (only once)
+ data.keynrs[i] = getInputRowMeta().indexOfValue(key.getKeyStream());
+ if (data.keynrs[i] < 0) {
+ throw new HopTransformException(
+ BaseMessages.getString(
+ PKG,
+ "StreamLookup.Log.FieldNotFound",
+ key.getKeyStream(),
+ getInputRowMeta().getString(r)));
+ } else {
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG, "StreamLookup.Log.FieldInfo", key.getKeyStream(), "" +
data.keynrs[i]));
+ }
+ }
+
+
data.lookupMeta.addValueMeta(getInputRowMeta().getValueMeta(data.keynrs[i]).clone());
+
+ // If we have binary storage data coming in, we convert it to normal
data storage.
+ // The storage in the lookup data store is also normal data storage.
+ //
+ data.convertKeysToNative[i] =
+
getInputRowMeta().getValueMeta(data.keynrs[i]).isStorageBinaryString();
+ }
+
+ data.outputRowMeta = getInputRowMeta().clone();
+ meta.getFields(
+ data.outputRowMeta,
+ getTransformName(),
+ new IRowMeta[] {data.infoMeta},
+ null,
+ this,
+ metadataProvider);
+
+ // Handle the NULL values (not found...)
+ handleNullIf();
+ }
+
+ private void calculateDefaultValues() {
+ // Calculate the default values by doing variable replacement.
+ //
+ data.valueDefault = new String[meta.getLookup().getReturnValues().size()];
+ for (int i = 0; i < data.valueDefault.length; i++) {
+ StreamLookupMeta.ReturnValue returnValue =
meta.getLookup().getReturnValues().get(i);
+
+ // We want to do this variable replacement only once since it's a
relatively slow operation.
+ //
+ data.valueDefault[i] = resolve(returnValue.getValueDefault());
+ }
+ }
+
@Override
public boolean init() {
-
if (super.init()) {
data.readLookupValues = true;
-
return true;
}
-
return false;
}
diff --git
a/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupDialog.java
b/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupDialog.java
index 34d7d8476e..d9fe74c08f 100644
---
a/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupDialog.java
+++
b/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupDialog.java
@@ -18,6 +18,7 @@
package org.apache.hop.pipeline.transforms.streamlookup;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IRowMeta;
@@ -87,7 +88,7 @@ public class StreamLookupDialog extends BaseTransformDialog {
.get(e -> get())
.custom(
BaseMessages.getString(PKG,
"StreamLookupDialog.GetLookupFields.Button"),
- e -> getlookup())
+ e -> getLookup())
.build();
ModifyListener lsMod = e -> input.setChanged();
@@ -134,7 +135,6 @@ public class StreamLookupDialog extends BaseTransformDialog
{
wlKey.setLayoutData(fdlKey);
int nrKeyCols = 2;
- int nrKeyRows = (input.getKeystream() != null ?
input.getKeystream().length : 1);
ciKey = new ColumnInfo[nrKeyCols];
ciKey[0] =
@@ -156,7 +156,7 @@ public class StreamLookupDialog extends BaseTransformDialog
{
shell,
SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL |
SWT.H_SCROLL,
ciKey,
- nrKeyRows,
+ 1,
lsMod,
props);
@@ -240,7 +240,6 @@ public class StreamLookupDialog extends BaseTransformDialog
{
wlReturn.setLayoutData(fdlReturn);
int upInsCols = 4;
- int upInsRows = (input.getValue() != null ? input.getValue().length : 1);
ciReturn = new ColumnInfo[upInsCols];
ciReturn[0] =
@@ -272,7 +271,7 @@ public class StreamLookupDialog extends BaseTransformDialog
{
shell,
SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL |
SWT.H_SCROLL,
ciReturn,
- upInsRows,
+ 1,
lsMod,
props);
@@ -360,37 +359,26 @@ public class StreamLookupDialog extends
BaseTransformDialog {
logDebug(BaseMessages.getString(PKG,
"StreamLookupDialog.Log.GettingKeyInfo"));
}
- if (input.getKeystream() != null) {
- for (int i = 0; i < input.getKeystream().length; i++) {
- TableItem item = wKey.table.getItem(i);
- if (input.getKeystream()[i] != null) {
- item.setText(1, input.getKeystream()[i]);
- }
- if (input.getKeylookup()[i] != null) {
- item.setText(2, input.getKeylookup()[i]);
- }
- }
+ for (StreamLookupMeta.MatchKey matchKey :
input.getLookup().getMatchKeys()) {
+ TableItem item = new TableItem(wKey.table, SWT.NONE);
+ item.setText(1, Const.NVL(matchKey.getKeyStream(), ""));
+ item.setText(2, Const.NVL(matchKey.getKeyLookup(), ""));
}
-
- if (input.getValue() != null) {
- for (int i = 0; i < input.getValue().length; i++) {
- TableItem item = wReturn.table.getItem(i);
- if (input.getValue()[i] != null) {
- item.setText(1, input.getValue()[i]);
- }
- if (input.getValueName()[i] != null
- && !input.getValueName()[i].equals(input.getValue()[i])) {
- item.setText(2, input.getValueName()[i]);
- }
- if (input.getValueDefault()[i] != null) {
- item.setText(3, input.getValueDefault()[i]);
- }
- item.setText(4,
ValueMetaFactory.getValueMetaName(input.getValueDefaultType()[i]));
+ wKey.optimizeTableView();
+
+ for (StreamLookupMeta.ReturnValue returnValue :
input.getLookup().getReturnValues()) {
+ TableItem item = new TableItem(wReturn.table, SWT.NONE);
+ item.setText(1, Const.NVL(returnValue.getValue(), ""));
+ if (StringUtils.isNotEmpty(returnValue.getValueName())
+ &&
!returnValue.getValueName().equalsIgnoreCase(returnValue.getValue())) {
+ item.setText(2, returnValue.getValueName());
}
+ item.setText(3, Const.NVL(returnValue.getValueDefault(), ""));
+ item.setText(4,
ValueMetaFactory.getValueMetaName(returnValue.getValueDefaultType()));
}
+ wReturn.optimizeTableView();
- IStream infoStream = input.getTransformIOMeta().getInfoStreams().get(0);
- wTransform.setText(Const.NVL(infoStream.getTransformName(), ""));
+ wTransform.setText(Const.NVL(input.getSourceTransformName(), ""));
boolean isPreserveMemory = input.isMemoryPreservationActive();
wPreserveMemory.setSelection(isPreserveMemory);
@@ -403,11 +391,6 @@ public class StreamLookupDialog extends
BaseTransformDialog {
// but user will be forced to choose only one option later.
wSortedList.setSelection(input.isUsingSortedList());
wIntegerPair.setSelection(input.isUsingIntegerPair());
-
- wKey.setRowNums();
- wKey.optWidth(true);
- wReturn.setRowNums();
- wReturn.optWidth(true);
}
private void cancel() {
@@ -421,40 +404,36 @@ public class StreamLookupDialog extends
BaseTransformDialog {
return;
}
- int nrkeys = wKey.nrNonEmpty();
- int nrvalues = wReturn.nrNonEmpty();
- input.allocate(nrkeys, nrvalues);
input.setMemoryPreservationActive(wPreserveMemory.getSelection());
input.setUsingSortedList(wSortedList.getSelection());
input.setUsingIntegerPair(wIntegerPair.getSelection());
-
- if (log.isDebug()) {
- logDebug(BaseMessages.getString(PKG, "StreamLookupDialog.Log.FoundKeys",
nrkeys + ""));
+ StreamLookupMeta.Lookup lookup = input.getLookup();
+ lookup.getMatchKeys().clear();
+ for (TableItem item : wKey.getNonEmptyItems()) {
+ StreamLookupMeta.MatchKey matchKey = new StreamLookupMeta.MatchKey();
+ lookup.getMatchKeys().add(matchKey);
+
+ matchKey.setKeyStream(item.getText(1));
+ matchKey.setKeyLookup(item.getText(2));
}
- for (int i = 0; i < nrkeys; i++) {
- TableItem item = wKey.getNonEmpty(i);
- input.getKeystream()[i] = item.getText(1);
- input.getKeylookup()[i] = item.getText(2);
- }
-
- if (log.isDebug()) {
- logDebug(BaseMessages.getString(PKG,
"StreamLookupDialog.Log.FoundFields", nrvalues + ""));
- }
+ lookup.getReturnValues().clear();
+ for (TableItem item : wReturn.getNonEmptyItems()) {
+ StreamLookupMeta.ReturnValue returnValue = new
StreamLookupMeta.ReturnValue();
+ lookup.getReturnValues().add(returnValue);
- for (int i = 0; i < nrvalues; i++) {
- TableItem item = wReturn.getNonEmpty(i);
- input.getValue()[i] = item.getText(1);
- input.getValueName()[i] = item.getText(2);
- if (Utils.isEmpty(input.getValueName()[i])) {
- input.getValueName()[i] = input.getValue()[i];
+ returnValue.setValue(item.getText(1));
+ returnValue.setValueName(item.getText(2));
+ if (Utils.isEmpty(returnValue.getValueName())) {
+ returnValue.setValueName(returnValue.getValue());
}
- input.getValueDefault()[i] = item.getText(3);
- input.getValueDefaultType()[i] =
ValueMetaFactory.getIdForValueMeta(item.getText(4));
+ returnValue.setValueDefault(item.getText(3));
+
returnValue.setValueDefaultType(ValueMetaFactory.getIdForValueMeta(item.getText(4)));
}
- IStream infoStream = input.getTransformIOMeta().getInfoStreams().get(0);
-
infoStream.setTransformMeta(pipelineMeta.findTransform(wTransform.getText()));
+ input.setSourceTransformName(wTransform.getText());
+ input.searchInfoAndTargetTransforms(pipelineMeta.getTransforms());
+ IStream infoStream =
input.getTransformIOMeta().getInfoStreams().getFirst();
if (infoStream.getTransformMeta() == null) {
MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
if (Utils.isEmpty(wTransform.getText())) {
@@ -533,7 +512,7 @@ public class StreamLookupDialog extends BaseTransformDialog
{
}
}
- private void getlookup() {
+ private void getLookup() {
try {
String transformFrom = wTransform.getText();
if (!Utils.isEmpty(transformFrom)) {
diff --git
a/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupMeta.java
b/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupMeta.java
index 3f8b55c233..5a8fe0ab1e 100644
---
a/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupMeta.java
+++
b/plugins/transforms/streamlookup/src/main/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupMeta.java
@@ -17,22 +17,22 @@
package org.apache.hop.pipeline.transforms.streamlookup;
+import java.util.ArrayList;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.Const;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.exception.HopXmlException;
-import org.apache.hop.core.injection.AfterInjection;
-import org.apache.hop.core.injection.Injection;
-import org.apache.hop.core.injection.InjectionSupported;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaBase;
import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.variables.IVariables;
-import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
@@ -43,9 +43,7 @@ import org.apache.hop.pipeline.transform.stream.IStream;
import org.apache.hop.pipeline.transform.stream.IStream.StreamType;
import org.apache.hop.pipeline.transform.stream.Stream;
import org.apache.hop.pipeline.transform.stream.StreamIcon;
-import org.w3c.dom.Node;
-@InjectionSupported(localizationPrefix = "StreamLookupMeta.Injection.")
@Transform(
id = "StreamLookup",
image = "streamlookup.svg",
@@ -54,149 +52,68 @@ import org.w3c.dom.Node;
categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Lookup",
keywords = "i18n::StreamLookupMeta.keyword",
documentationUrl = "/pipeline/transforms/streamlookup.html")
+@Getter
+@Setter
public class StreamLookupMeta extends BaseTransformMeta<StreamLookup,
StreamLookupData> {
private static final Class<?> PKG = StreamLookupMeta.class;
public static final String CONST_SPACES = " ";
- /** fields in input streams with which we look up values */
- @Injection(name = "KEY_STREAM")
- private String[] keystream;
-
- /** fields in lookup stream with which we look up values */
- @Injection(name = "KEY_LOOKUP")
- private String[] keylookup;
-
- /** return these field values from lookup */
- @Injection(name = "RETRIEVE_VALUE")
- private String[] value;
-
- /** rename to this after lookup */
- @Injection(name = "RETRIEVE_VALUE_NAME")
- private String[] valueName;
-
- /** default value in case not found... */
- @Injection(name = "RETRIEVE_VALUE_DEFAULT")
- private String[] valueDefault;
-
- /** type of default value */
- @Injection(name = "RETRIEVE_DEFAULT_TYPE")
- private int[] valueDefaultType;
+ @HopMetadataProperty(
+ key = "from",
+ injectionKey = "SOURCE_TRANSFORM",
+ injectionKeyDescription = "StreamLookupMeta.Injection.SOURCE_TRANSFORM")
+ private String sourceTransformName;
/** Indicate that the input is considered sorted! */
+ @HopMetadataProperty(
+ key = "input_sorted",
+ injectionKey = "INPUT_SORTED",
+ injectionKeyDescription = "StreamLookupMeta.Injection.INPUT_SORTED")
private boolean inputSorted;
/** Indicate that we need to preserve memory by serializing objects */
- @Injection(name = "PRESERVE_MEMORY")
+ @HopMetadataProperty(
+ key = "preserve_memory",
+ injectionKey = "PRESERVE_MEMORY",
+ injectionKeyDescription = "StreamLookupMeta.Injection.PRESERVE_MEMORY")
private boolean memoryPreservationActive;
/** Indicate that we want to use a sorted list vs. a hashtable */
- @Injection(name = "SORTED_LIST")
+ @HopMetadataProperty(
+ key = "sorted_list",
+ injectionKey = "SORTED_LIST",
+ injectionKeyDescription = "StreamLookupMeta.Injection.SORTED_LIST")
private boolean usingSortedList;
/** The content of the key and lookup is a single Integer (long) */
- @Injection(name = "INTEGER_PAIR")
+ @HopMetadataProperty(
+ key = "integer_pair",
+ injectionKey = "INTEGER_PAIR",
+ injectionKeyDescription = "StreamLookupMeta.Injection.INTEGER_PAIR")
private boolean usingIntegerPair;
- public StreamLookupMeta() {
- super(); // allocate BaseTransformMeta
- }
+ @HopMetadataProperty(key = "lookup")
+ private Lookup lookup;
- @Override
- public void loadXml(Node transformNode, IHopMetadataProvider
metadataProvider)
- throws HopXmlException {
- readData(transformNode);
+ public StreamLookupMeta() {
+ super();
+ memoryPreservationActive = true;
+ lookup = new Lookup();
}
- public void allocate(int nrkeys, int nrvalues) {
- setKeystream(new String[nrkeys]);
- setKeylookup(new String[nrkeys]);
-
- setValue(new String[nrvalues]);
- setValueName(new String[nrvalues]);
- setValueDefault(new String[nrvalues]);
- setValueDefaultType(new int[nrvalues]);
+ public StreamLookupMeta(StreamLookupMeta m) {
+ this();
+ this.sourceTransformName = m.sourceTransformName;
+ this.inputSorted = m.inputSorted;
+ this.memoryPreservationActive = m.memoryPreservationActive;
+ this.usingSortedList = m.usingSortedList;
+ this.usingIntegerPair = m.usingIntegerPair;
+ this.lookup = new Lookup(m.lookup);
}
@Override
public Object clone() {
- StreamLookupMeta retval = (StreamLookupMeta) super.clone();
- ITransformIOMeta thisTransformIO = getTransformIOMeta();
- ITransformIOMeta thatTransformIO = retval.getTransformIOMeta();
- if (thisTransformIO != null
- && thisTransformIO.getInfoStreams() != null
- && thatTransformIO != null
- && thatTransformIO.getInfoStreams() != null) {
- List<IStream> thisInfoStream = thisTransformIO.getInfoStreams();
- List<IStream> thatInfoStream = thatTransformIO.getInfoStreams();
-
thatInfoStream.get(0).setStreamType(thisInfoStream.get(0).getStreamType());
-
thatInfoStream.get(0).setTransformMeta(thisInfoStream.get(0).getTransformMeta());
-
thatInfoStream.get(0).setDescription(thisInfoStream.get(0).getDescription());
-
thatInfoStream.get(0).setStreamIcon(thisInfoStream.get(0).getStreamIcon());
- thatInfoStream.get(0).setSubject(thisInfoStream.get(0).getSubject());
- }
-
- int nrkeys = keystream.length;
- int nrvals = value.length;
- retval.allocate(nrkeys, nrvals);
- System.arraycopy(keystream, 0, retval.keystream, 0, nrkeys);
- System.arraycopy(keylookup, 0, retval.keylookup, 0, nrkeys);
- System.arraycopy(value, 0, retval.value, 0, nrvals);
- System.arraycopy(valueName, 0, retval.valueName, 0, nrvals);
- System.arraycopy(valueDefault, 0, retval.valueDefault, 0, nrvals);
- System.arraycopy(valueDefaultType, 0, retval.valueDefaultType, 0, nrvals);
- return retval;
- }
-
- private void readData(Node transformNode) throws HopXmlException {
- try {
- String dtype;
- int nrkeys;
- int nrvalues;
-
- String lookupFromTransformName = XmlHandler.getTagValue(transformNode,
"from");
- IStream infoStream = getTransformIOMeta().getInfoStreams().get(0);
- infoStream.setSubject(lookupFromTransformName);
-
-
setInputSorted("Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"input_sorted")));
- setMemoryPreservationActive(
- "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"preserve_memory")));
- setUsingSortedList(
- "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"sorted_list")));
- setUsingIntegerPair(
- "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"integer_pair")));
-
- Node lookup = XmlHandler.getSubNode(transformNode, "lookup");
- nrkeys = XmlHandler.countNodes(lookup, "key");
- nrvalues = XmlHandler.countNodes(lookup, "value");
-
- allocate(nrkeys, nrvalues);
-
- for (int i = 0; i < nrkeys; i++) {
- Node knode = XmlHandler.getSubNodeByNr(lookup, "key", i);
-
- getKeystream()[i] = XmlHandler.getTagValue(knode, "name");
- getKeylookup()[i] = XmlHandler.getTagValue(knode, "field");
- }
-
- for (int i = 0; i < nrvalues; i++) {
- Node vnode = XmlHandler.getSubNodeByNr(lookup, "value", i);
-
- getValue()[i] = XmlHandler.getTagValue(vnode, "name");
- getValueName()[i] = XmlHandler.getTagValue(vnode, "rename");
- if (getValueName()[i] == null) {
- getValueName()[i] = getValue()[i]; // default: same name to return!
- }
-
- getValueDefault()[i] = XmlHandler.getTagValue(vnode, "default");
- dtype = XmlHandler.getTagValue(vnode, "type");
- getValueDefaultType()[i] = ValueMetaFactory.getIdForValueMeta(dtype);
- }
- } catch (Exception e) {
- throw new HopXmlException(
- BaseMessages.getString(
- PKG,
"StreamLookupMeta.Exception.UnableToLoadTransformMetaFromXML"),
- e);
- }
+ return new StreamLookupMeta(this);
}
@Override
@@ -207,15 +124,6 @@ public class StreamLookupMeta extends
BaseTransformMeta<StreamLookup, StreamLook
}
}
- @Override
- public void setDefault() {
- setMemoryPreservationActive(true);
- setUsingSortedList(false);
- setUsingIntegerPair(false);
-
- allocate(0, 0);
- }
-
@Override
public void getFields(
IRowMeta row,
@@ -226,24 +134,24 @@ public class StreamLookupMeta extends
BaseTransformMeta<StreamLookup, StreamLook
IHopMetadataProvider metadataProvider)
throws HopTransformException {
if (info != null && info.length == 1 && info[0] != null) {
- for (int i = 0; i < getValueName().length; i++) {
- IValueMeta valueMeta = info[0].searchValueMeta(getValue()[i]);
+ for (ReturnValue value : lookup.returnValues) {
+ IValueMeta valueMeta = info[0].searchValueMeta(value.getValue());
if (valueMeta != null) {
// Configuration error/missing resources...
- valueMeta.setName(getValueName()[i]);
+ valueMeta.setName(value.getValueName());
valueMeta.setOrigin(origin);
row.addValueMeta(valueMeta);
} else {
throw new HopTransformException(
BaseMessages.getString(
- PKG, "StreamLookupMeta.Exception.ReturnValueCanNotBeFound",
getValue()[i]));
+ PKG, "StreamLookupMeta.Exception.ReturnValueCanNotBeFound",
value.getValue()));
}
}
} else {
- for (int i = 0; i < getValueName().length; i++) {
+ for (ReturnValue value : lookup.returnValues) {
try {
IValueMeta v =
- ValueMetaFactory.createValueMeta(getValueName()[i],
getValueDefaultType()[i]);
+ ValueMetaFactory.createValueMeta(value.getValueName(),
value.getValueDefaultType());
v.setOrigin(origin);
row.addValueMeta(v);
} catch (Exception e) {
@@ -253,44 +161,6 @@ public class StreamLookupMeta extends
BaseTransformMeta<StreamLookup, StreamLook
}
}
- @Override
- public String getXml() {
- StringBuilder retval = new StringBuilder();
-
- IStream infoStream = getTransformIOMeta().getInfoStreams().get(0);
- retval.append(" ").append(XmlHandler.addTagValue("from",
infoStream.getTransformName()));
- retval.append(" ").append(XmlHandler.addTagValue("input_sorted",
isInputSorted()));
- retval
- .append(" ")
- .append(XmlHandler.addTagValue("preserve_memory",
isMemoryPreservationActive()));
- retval.append(" ").append(XmlHandler.addTagValue("sorted_list",
isUsingSortedList()));
- retval.append(" ").append(XmlHandler.addTagValue("integer_pair",
isUsingIntegerPair()));
-
- retval.append(" <lookup>").append(Const.CR);
- for (int i = 0; i < getKeystream().length; i++) {
- retval.append(" <key>").append(Const.CR);
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("name",
getKeystream()[i]));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("field",
getKeylookup()[i]));
- retval.append(" </key>").append(Const.CR);
- }
-
- for (int i = 0; i < getValue().length; i++) {
- retval.append(" <value>").append(Const.CR);
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("name",
getValue()[i]));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("rename",
getValueName()[i]));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("default",
getValueDefault()[i]));
- retval
- .append(CONST_SPACES)
- .append(
- XmlHandler.addTagValue(
- "type",
ValueMetaFactory.getValueMetaName(getValueDefaultType()[i])));
- retval.append(" </value>").append(Const.CR);
- }
- retval.append(" </lookup>").append(Const.CR);
-
- return retval.toString();
- }
-
@Override
public void check(
List<ICheckResult> remarks,
@@ -318,10 +188,10 @@ public class StreamLookupMeta extends
BaseTransformMeta<StreamLookup, StreamLook
// Starting from selected fields in ...
// Check the fields from the previous stream!
- for (String aKeystream : getKeystream()) {
- int idx = prev.indexOfValue(aKeystream);
+ for (MatchKey key : lookup.matchKeys) {
+ int idx = prev.indexOfValue(key.keyStream);
if (idx < 0) {
- errorMessage += "\t\t" + aKeystream + Const.CR;
+ errorMessage += "\t\t" + key.keyStream + Const.CR;
errorFound = true;
}
}
@@ -367,10 +237,10 @@ public class StreamLookupMeta extends
BaseTransformMeta<StreamLookup, StreamLook
boolean errorFound = false;
// Check the fields from the lookup stream!
- for (String aKeylookup : getKeylookup()) {
- int idx = info.indexOfValue(aKeylookup);
+ for (MatchKey key : lookup.matchKeys) {
+ int idx = info.indexOfValue(key.keyLookup);
if (idx < 0) {
- errorMessage += "\t\t" + aKeylookup + Const.CR;
+ errorMessage += "\t\t" + key.keyLookup + Const.CR;
errorFound = true;
}
}
@@ -394,10 +264,10 @@ public class StreamLookupMeta extends
BaseTransformMeta<StreamLookup, StreamLook
}
// Check the values to retrieve from the lookup stream!
- for (String aValue : getValue()) {
- int idx = info.indexOfValue(aValue);
+ for (ReturnValue value : lookup.returnValues) {
+ int idx = info.indexOfValue(value.getValue());
if (idx < 0) {
- errorMessage += "\t\t" + aValue + Const.CR;
+ errorMessage += "\t\t" + value.getValue() + Const.CR;
errorFound = true;
}
}
@@ -431,7 +301,7 @@ public class StreamLookupMeta extends
BaseTransformMeta<StreamLookup, StreamLook
}
// See if the source transform is filled in!
- IStream infoStream = getTransformIOMeta().getInfoStreams().get(0);
+ IStream infoStream = getTransformIOMeta().getInfoStreams().getFirst();
if (infoStream.getTransformMeta() == null) {
cr =
new CheckResult(
@@ -510,14 +380,13 @@ public class StreamLookupMeta extends
BaseTransformMeta<StreamLookup, StreamLook
if (ioMeta == null) {
ioMeta = new TransformIOMeta(true, true, false, false, false, false);
-
IStream stream =
new Stream(
StreamType.INFO,
null,
BaseMessages.getString(PKG,
"StreamLookupMeta.InfoStream.Description"),
StreamIcon.INFO,
- null);
+ sourceTransformName);
ioMeta.addStream(stream);
setTransformIOMeta(ioMeta);
}
@@ -530,163 +399,99 @@ public class StreamLookupMeta extends
BaseTransformMeta<StreamLookup, StreamLook
// Do nothing, don't reset as there is no need to do this.
}
- /**
- * @return Returns the inputSorted.
- */
- public boolean isInputSorted() {
- return inputSorted;
- }
-
- /**
- * @param inputSorted The inputSorted to set.
- */
- public void setInputSorted(boolean inputSorted) {
- this.inputSorted = inputSorted;
- }
-
- /**
- * @return Returns the keylookup.
- */
- public String[] getKeylookup() {
- return keylookup;
- }
-
- /**
- * @param keylookup The keylookup to set.
- */
- public void setKeylookup(String[] keylookup) {
- this.keylookup = keylookup;
- }
-
- /**
- * @return Returns the keystream.
- */
- public String[] getKeystream() {
- return keystream;
- }
-
- /**
- * @param keystream The keystream to set.
- */
- public void setKeystream(String[] keystream) {
- this.keystream = keystream;
- }
-
- /**
- * @return Returns the value.
- */
- public String[] getValue() {
- return value;
- }
-
- /**
- * @param value The value to set.
- */
- public void setValue(String[] value) {
- this.value = value;
- }
-
- /**
- * @return Returns the valueDefault.
- */
- public String[] getValueDefault() {
- return valueDefault;
- }
-
- /**
- * @param valueDefault The valueDefault to set.
- */
- public void setValueDefault(String[] valueDefault) {
- this.valueDefault = valueDefault;
- }
-
- /**
- * @return Returns the valueDefaultType.
- */
- public int[] getValueDefaultType() {
- return valueDefaultType;
- }
-
- /**
- * @param valueDefaultType The valueDefaultType to set.
- */
- public void setValueDefaultType(int[] valueDefaultType) {
- this.valueDefaultType = valueDefaultType;
- }
-
- /**
- * @return Returns the valueName.
- */
- public String[] getValueName() {
- return valueName;
- }
-
- /**
- * @param valueName The valueName to set.
- */
- public void setValueName(String[] valueName) {
- this.valueName = valueName;
- }
-
- public boolean isMemoryPreservationActive() {
- return memoryPreservationActive;
- }
-
- public void setMemoryPreservationActive(boolean memoryPreservationActive) {
- this.memoryPreservationActive = memoryPreservationActive;
- }
-
- public boolean isUsingSortedList() {
- return usingSortedList;
- }
-
- public void setUsingSortedList(boolean usingSortedList) {
- this.usingSortedList = usingSortedList;
- }
-
- /**
- * @return the usingIntegerPair
- */
- public boolean isUsingIntegerPair() {
- return usingIntegerPair;
- }
+ @Getter
+ @Setter
+ public static class Lookup {
+ @HopMetadataProperty(
+ key = "key",
+ injectionGroupKey = "KEYS",
+ injectionGroupDescription = "StreamLookupMeta.Injection.KEYS")
+ private List<MatchKey> matchKeys;
+
+ @HopMetadataProperty(
+ key = "value",
+ injectionGroupKey = "VALUES",
+ injectionGroupDescription = "StreamLookupMeta.Injection.VALUES")
+ private List<ReturnValue> returnValues;
+
+ public Lookup() {
+ matchKeys = new ArrayList<>();
+ returnValues = new ArrayList<>();
+ }
- /**
- * @param usingIntegerPair the usingIntegerPair to set
- */
- public void setUsingIntegerPair(boolean usingIntegerPair) {
- this.usingIntegerPair = usingIntegerPair;
+ public Lookup(Lookup l) {
+ this();
+ l.matchKeys.forEach(k -> matchKeys.add(new MatchKey(k)));
+ l.returnValues.forEach(v -> returnValues.add(new ReturnValue(v)));
+ }
}
- /**
- * If we use injection we can have different arrays lengths. We need
synchronize them for
- * consistency behavior with UI
- */
- @AfterInjection
- public void afterInjectionSynchronization() {
- if (value == null || value.length == 0) {
- return;
- }
- int nrFields = value.length;
- if (valueDefaultType.length < nrFields) {
- int[] newValueDefaultType = new int[nrFields];
- System.arraycopy(valueDefaultType, 0, newValueDefaultType, 0,
valueDefaultType.length);
- for (int i = valueDefaultType.length; i < newValueDefaultType.length;
i++) {
- newValueDefaultType[i] =
- -1; // set a undefined value (<0). It will be correct processed in
a handleNullIf method
- }
- valueDefaultType = newValueDefaultType;
- }
- if (valueName.length < nrFields) {
- String[] newValueName = new String[nrFields];
- System.arraycopy(valueName, 0, newValueName, 0, valueName.length);
- valueName = newValueName;
+ @Getter
+ @Setter
+ public static class MatchKey {
+ /** fields in input streams with which we look up values */
+ @HopMetadataProperty(
+ key = "name",
+ injectionKey = "KEY_STREAM",
+ injectionKeyDescription = "StreamLookupMeta.Injection.KEY_STREAM")
+ private String keyStream;
+
+ /** fields in lookup stream with which we look up values */
+ @HopMetadataProperty(
+ key = "field",
+ injectionKey = "KEY_LOOKUP",
+ injectionKeyDescription = "StreamLookupMeta.Injection.KEY_LOOKUP")
+ private String keyLookup;
+
+ public MatchKey() {}
+
+ public MatchKey(MatchKey k) {
+ this();
+ this.keyStream = k.keyStream;
+ this.keyLookup = k.keyLookup;
}
+ }
- if (valueDefault.length < nrFields) {
- String[] newValueDefault = new String[nrFields];
- System.arraycopy(valueDefault, 0, newValueDefault, 0,
valueDefault.length);
- valueDefault = newValueDefault;
+ @Getter
+ @Setter
+ public static class ReturnValue {
+ /** return these field values from lookup */
+ @HopMetadataProperty(
+ key = "name",
+ injectionKey = "RETRIEVE_VALUE",
+ injectionKeyDescription = "StreamLookupMeta.Injection.RETRIEVE_VALUE")
+ private String value;
+
+ /** rename to this after lookup */
+ @HopMetadataProperty(
+ key = "rename",
+ injectionKey = "RETRIEVE_VALUE_NAME",
+ injectionKeyDescription =
"StreamLookupMeta.Injection.RETRIEVE_VALUE_NAME")
+ private String valueName;
+
+ /** default value in case not found... */
+ @HopMetadataProperty(
+ key = "default",
+ injectionKey = "RETRIEVE_VALUE_DEFAULT",
+ injectionKeyDescription =
"StreamLookupMeta.Injection.RETRIEVE_VALUE_DEFAULT")
+ private String valueDefault;
+
+ /** type of default value */
+ @HopMetadataProperty(
+ key = "type",
+ intCodeConverter = ValueMetaBase.ValueTypeCodeConverter.class,
+ injectionKey = "RETRIEVE_DEFAULT_TYPE",
+ injectionKeyDescription =
"StreamLookupMeta.Injection.RETRIEVE_DEFAULT_TYPE")
+ private int valueDefaultType;
+
+ public ReturnValue() {}
+
+ public ReturnValue(ReturnValue v) {
+ this();
+ this.value = v.value;
+ this.valueName = v.valueName;
+ this.valueDefault = v.valueDefault;
+ this.valueDefaultType = v.valueDefaultType;
}
}
}
diff --git
a/plugins/transforms/streamlookup/src/main/resources/org/apache/hop/pipeline/transforms/streamlookup/messages/messages_en_US.properties
b/plugins/transforms/streamlookup/src/main/resources/org/apache/hop/pipeline/transforms/streamlookup/messages/messages_en_US.properties
index 5493e858f7..212a5cefae 100644
---
a/plugins/transforms/streamlookup/src/main/resources/org/apache/hop/pipeline/transforms/streamlookup/messages/messages_en_US.properties
+++
b/plugins/transforms/streamlookup/src/main/resources/org/apache/hop/pipeline/transforms/streamlookup/messages/messages_en_US.properties
@@ -88,3 +88,7 @@ StreamLookupMeta.Injection.RETRIEVE_VALUE_DEFAULT=The default
value if no value
StreamLookupMeta.Injection.RETRIEVE_VALUE_NAME=After a lookup, the list of
values retrieved will be used instead of the existing values.
StreamLookupMeta.Injection.SORTED_LIST=Indicates that the input is sorted.
StreamLookupMeta.keyword=stream,lookup
+StreamLookupMeta.Injection.SOURCE_TRANSFORM=The name of the source transform
to read from.
+StreamLookupMeta.Injection.INPUT_SORTED=Is the input considered sorted?
+StreamLookupMeta.Injection.KEYS=Match keys
+StreamLookupMeta.Injection.VALUES=Return values
diff --git
a/plugins/transforms/streamlookup/src/test/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupMetaTest.java
b/plugins/transforms/streamlookup/src/test/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupMetaTest.java
index 05e0b4bfde..1a4741c8c4 100644
---
a/plugins/transforms/streamlookup/src/test/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupMetaTest.java
+++
b/plugins/transforms/streamlookup/src/test/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupMetaTest.java
@@ -17,129 +17,57 @@
package org.apache.hop.pipeline.transforms.streamlookup;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotSame;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import static org.junit.jupiter.api.Assertions.assertTrue;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.hop.core.HopEnvironment;
-import org.apache.hop.core.exception.HopException;
-import org.apache.hop.core.plugins.PluginRegistry;
-import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
-import org.apache.hop.pipeline.transform.ITransformMeta;
-import org.apache.hop.pipeline.transform.TransformMeta;
-import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
-import org.apache.hop.pipeline.transforms.loadsave.initializer.IInitializer;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.ArrayLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.IntLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.PrimitiveIntArrayLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.StringLoadSaveValidator;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaPluginType;
+import org.apache.hop.pipeline.transform.TransformSerializationTestUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-class StreamLookupMetaTest implements IInitializer<ITransformMeta> {
- LoadSaveTester loadSaveTester;
- Class<StreamLookupMeta> testMetaClass = StreamLookupMeta.class;
-
- @RegisterExtension
- static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+class StreamLookupMetaTest {
@BeforeEach
- void setUpLoadSave() throws Exception {
- HopEnvironment.init();
- PluginRegistry.init();
- List<String> attributes =
- Arrays.asList(
- "inputSorted",
- "memoryPreservationActive",
- "usingSortedList",
- "usingIntegerPair",
- "keystream",
- "keylookup",
- "value",
- "valueName",
- "valueDefault",
- "valueDefaultType");
-
- IFieldLoadSaveValidator<String[]> stringArrayLoadSaveValidator =
- new ArrayLoadSaveValidator<>(new StringLoadSaveValidator(), 5);
-
- Map<String, IFieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<>();
- attrValidatorMap.put("keystream", stringArrayLoadSaveValidator);
- attrValidatorMap.put("keylookup", stringArrayLoadSaveValidator);
- attrValidatorMap.put("value", stringArrayLoadSaveValidator);
- attrValidatorMap.put("valueName", stringArrayLoadSaveValidator);
- attrValidatorMap.put("valueDefault", stringArrayLoadSaveValidator);
- attrValidatorMap.put(
- "valueDefaultType", new PrimitiveIntArrayLoadSaveValidator(new
IntLoadSaveValidator(7), 5));
-
- Map<String, IFieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<>();
-
- loadSaveTester =
- new LoadSaveTester(
- testMetaClass,
- attributes,
- new HashMap<>(),
- new HashMap<>(),
- attrValidatorMap,
- typeValidatorMap,
- this);
- }
-
- // Call the allocate method on the LoadSaveTester meta class
- @Override
- public void modify(ITransformMeta someMeta) {
- if (someMeta instanceof StreamLookupMeta) {
- ((StreamLookupMeta) someMeta).allocate(5, 5);
- }
- }
-
- @Test
- void testSerialization() throws HopException {
- loadSaveTester.testSerialization();
+ void beforeEach() throws Exception {
+ ValueMetaPluginType.getInstance().searchPlugins();
}
@Test
- void testCloneInfoTransforms() {
- StreamLookupMeta meta = new StreamLookupMeta();
- meta.setDefault();
-
- final String transformName = UUID.randomUUID().toString();
- TransformMeta infoTransform = mock(TransformMeta.class);
- when(infoTransform.getName()).thenReturn(transformName);
-
meta.getTransformIOMeta().getInfoStreams().get(0).setTransformMeta(infoTransform);
-
- StreamLookupMeta cloned = (StreamLookupMeta) meta.clone();
- assertEquals(
- transformName,
cloned.getTransformIOMeta().getInfoStreams().get(0).getTransformName());
- assertNotSame(
- meta.getTransformIOMeta().getInfoStreams().get(0),
- cloned.getTransformIOMeta().getInfoStreams().get(0));
- }
-
- @Test
- void testGetXml() {
- StreamLookupMeta streamLookupMeta = new StreamLookupMeta();
- streamLookupMeta.setKeystream(new String[] {"testKeyStreamValue"});
- streamLookupMeta.setKeylookup(new String[] {"testKeyLookupValue"});
- streamLookupMeta.setValue(new String[] {"testValue"});
- streamLookupMeta.setValueName(new String[] {});
- streamLookupMeta.setValueDefault(new String[] {});
- streamLookupMeta.setValueDefaultType(new int[] {});
-
- // run without exception
- streamLookupMeta.afterInjectionSynchronization();
- streamLookupMeta.getXml();
-
- assertEquals(streamLookupMeta.getKeystream().length,
streamLookupMeta.getValueName().length);
- assertEquals(streamLookupMeta.getKeystream().length,
streamLookupMeta.getValueDefault().length);
- assertEquals(
- streamLookupMeta.getKeystream().length,
streamLookupMeta.getValueDefaultType().length);
+ void testSerializationRoundTrip() throws Exception {
+ StreamLookupMeta meta =
+ TransformSerializationTestUtil.testSerialization(
+ "/stream-lookup.xml", StreamLookupMeta.class);
+
+ assertTrue(meta.isMemoryPreservationActive());
+ assertTrue(meta.isUsingIntegerPair());
+ assertTrue(meta.isUsingSortedList());
+
+ assertEquals(2, meta.getLookup().getMatchKeys().size());
+ StreamLookupMeta.MatchKey k = meta.getLookup().getMatchKeys().getFirst();
+ assertEquals("keyStream1", k.getKeyStream());
+ assertEquals("keyLookup1", k.getKeyLookup());
+
+ k = meta.getLookup().getMatchKeys().getLast();
+ assertEquals("keyStream2", k.getKeyStream());
+ assertEquals("keyLookup2", k.getKeyLookup());
+
+ assertEquals(3, meta.getLookup().getReturnValues().size());
+ StreamLookupMeta.ReturnValue v =
meta.getLookup().getReturnValues().getFirst();
+ assertEquals("returnValue1", v.getValue());
+ assertEquals("value1", v.getValueName());
+ assertTrue(StringUtils.isEmpty(v.getValueDefault()));
+ assertEquals(IValueMeta.TYPE_STRING, v.getValueDefaultType());
+
+ v = meta.getLookup().getReturnValues().get(1);
+ assertEquals("returnValue2", v.getValue());
+ assertEquals("value2", v.getValueName());
+ assertEquals("123", v.getValueDefault());
+ assertEquals(IValueMeta.TYPE_INTEGER, v.getValueDefaultType());
+
+ v = meta.getLookup().getReturnValues().getLast();
+ assertEquals("returnValue3", v.getValue());
+ assertEquals("value3", v.getValueName());
+ assertEquals("true", v.getValueDefault());
+ assertEquals(IValueMeta.TYPE_BOOLEAN, v.getValueDefaultType());
}
}
diff --git
a/plugins/transforms/streamlookup/src/test/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupTest.java
b/plugins/transforms/streamlookup/src/test/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupTest.java
deleted file mode 100644
index 8878a0df44..0000000000
---
a/plugins/transforms/streamlookup/src/test/java/org/apache/hop/pipeline/transforms/streamlookup/StreamLookupTest.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.pipeline.transforms.streamlookup;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doCallRealMethod;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.nullable;
-import static org.mockito.Mockito.when;
-
-import org.apache.hop.core.IRowSet;
-import org.apache.hop.core.QueueRowSet;
-import org.apache.hop.core.exception.HopException;
-import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.logging.ILoggingObject;
-import org.apache.hop.core.row.IRowMeta;
-import org.apache.hop.core.row.IValueMeta;
-import org.apache.hop.core.row.RowMeta;
-import org.apache.hop.core.row.value.ValueMetaString;
-import org.apache.hop.core.variables.IVariables;
-import org.apache.hop.metadata.api.IHopMetadataProvider;
-import org.apache.hop.pipeline.transform.TransformIOMeta;
-import org.apache.hop.pipeline.transform.TransformMeta;
-import org.apache.hop.pipeline.transform.stream.IStream;
-import org.apache.hop.pipeline.transform.stream.Stream;
-import org.apache.hop.pipeline.transform.stream.StreamIcon;
-import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-/**
- * Test for StreamLookup transform
- *
- * @see StreamLookup
- */
-class StreamLookupTest {
- private TransformMockHelper<StreamLookupMeta, StreamLookupData> smh;
-
- @BeforeEach
- void setUp() {
- smh = new TransformMockHelper<>("StreamLookup", StreamLookupMeta.class,
StreamLookupData.class);
- when(smh.logChannelFactory.create(any(), nullable(ILoggingObject.class)))
- .thenReturn(smh.iLogChannel);
- when(smh.pipeline.isRunning()).thenReturn(true);
- }
-
- @AfterEach
- void cleanUp() {
- smh.cleanUp();
- }
-
- private void convertDataToBinary(Object[][] data) {
- for (int i = 0; i < data.length; i++) {
- for (int j = 0; j < data[i].length; j++) {
- data[i][j] = ((String) data[i][j]).getBytes();
- }
- }
- }
-
- private IRowSet mockLookupRowSet(boolean binary) {
- final int storageType =
- binary ? IValueMeta.STORAGE_TYPE_BINARY_STRING :
IValueMeta.STORAGE_TYPE_NORMAL;
- Object[][] data = {{"Value1", "1"}, {"Value2", "2"}};
-
- if (binary) {
- convertDataToBinary(data);
- }
-
- IRowSet lookupRowSet = smh.getMockInputRowSet(data);
- doReturn("Lookup").when(lookupRowSet).getOriginTransformName();
- doReturn("StreamLookup").when(lookupRowSet).getDestinationTransformName();
-
- RowMeta lookupRowMeta = new RowMeta();
- ValueMetaString valueMeta = new ValueMetaString("Value");
- valueMeta.setStorageType(storageType);
- valueMeta.setStorageMetadata(new ValueMetaString());
- lookupRowMeta.addValueMeta(valueMeta);
- ValueMetaString idMeta = new ValueMetaString("Id");
- idMeta.setStorageType(storageType);
- idMeta.setStorageMetadata(new ValueMetaString());
- lookupRowMeta.addValueMeta(idMeta);
-
- doReturn(lookupRowMeta).when(lookupRowSet).getRowMeta();
-
- return lookupRowSet;
- }
-
- private IRowSet mockDataRowSet(boolean binary) {
- final int storageType =
- binary ? IValueMeta.STORAGE_TYPE_BINARY_STRING :
IValueMeta.STORAGE_TYPE_NORMAL;
- Object[][] data = {{"Name1", "1"}, {"Name2", "2"}};
-
- if (binary) {
- convertDataToBinary(data);
- }
-
- IRowSet dataRowSet = smh.getMockInputRowSet(data);
-
- RowMeta dataRowMeta = new RowMeta();
- ValueMetaString valueMeta = new ValueMetaString("Name");
- valueMeta.setStorageType(storageType);
- valueMeta.setStorageMetadata(new ValueMetaString());
- dataRowMeta.addValueMeta(valueMeta);
- ValueMetaString idMeta = new ValueMetaString("Id");
- idMeta.setStorageType(storageType);
- idMeta.setStorageMetadata(new ValueMetaString());
- dataRowMeta.addValueMeta(idMeta);
-
- doReturn(dataRowMeta).when(dataRowSet).getRowMeta();
-
- return dataRowSet;
- }
-
- private StreamLookupMeta mockProcessRowMeta(boolean memoryPreservationActive)
- throws HopTransformException {
- StreamLookupMeta meta = smh.iTransformMeta;
-
- TransformMeta lookupTransformMeta =
-
when(mock(TransformMeta.class).getName()).thenReturn("Lookup").getMock();
-
doReturn(lookupTransformMeta).when(smh.pipelineMeta).findTransform("Lookup");
-
- TransformIOMeta transformIOMeta = new TransformIOMeta(true, true, false,
false, false, false);
- transformIOMeta.addStream(
- new Stream(IStream.StreamType.INFO, lookupTransformMeta, null,
StreamIcon.INFO, null));
-
- doReturn(transformIOMeta).when(meta).getTransformIOMeta();
- doReturn(new String[] {"Id"}).when(meta).getKeylookup();
- doReturn(new String[] {"Id"}).when(meta).getKeystream();
- doReturn(new String[] {"Value"}).when(meta).getValue();
- doReturn(memoryPreservationActive).when(meta).isMemoryPreservationActive();
- doReturn(false).when(meta).isUsingSortedList();
- doReturn(false).when(meta).isUsingIntegerPair();
- doReturn(new int[] {-1}).when(meta).getValueDefaultType();
- doReturn(new String[] {""}).when(meta).getValueDefault();
- doReturn(new String[] {"Value"}).when(meta).getValueName();
- doReturn(new String[] {"Value"}).when(meta).getValue();
- doCallRealMethod()
- .when(meta)
- .getFields(
- nullable(IRowMeta.class),
- nullable(String.class),
- nullable(IRowMeta[].class),
- nullable(TransformMeta.class),
- nullable(IVariables.class),
- nullable(IHopMetadataProvider.class));
-
- return meta;
- }
-
- private void doTest(
- boolean memoryPreservationActive, boolean binaryLookupStream, boolean
binaryDataStream)
- throws HopException {
- StreamLookup transform =
- new StreamLookup(
- smh.transformMeta,
- smh.iTransformMeta,
- smh.iTransformData,
- 0,
- smh.pipelineMeta,
- smh.pipeline);
- transform.init();
- transform.addRowSetToInputRowSets(mockLookupRowSet(binaryLookupStream));
- transform.addRowSetToInputRowSets(mockDataRowSet(binaryDataStream));
- transform.addRowSetToOutputRowSets(new QueueRowSet());
-
- StreamLookupMeta meta = mockProcessRowMeta(memoryPreservationActive);
- StreamLookupData data = new StreamLookupData();
- data.readLookupValues = true;
-
- IRowSet outputRowSet = transform.getOutputRowSets().get(0);
-
- // Process rows and collect output
- int rowNumber = 0;
- String[] expectedOutput = {"Name", "", "Value"};
- while (transform.processRow()) {
- Object[] rowData = outputRowSet.getRow();
- if (rowData != null) {
- IRowMeta rowMeta = outputRowSet.getRowMeta();
- assertEquals(3, rowMeta.size(), "Output row is of wrong size");
- rowNumber++;
- // Verify output
- for (int valueIndex = 0; valueIndex < rowMeta.size(); valueIndex++) {
- String expectedValue = expectedOutput[valueIndex] + rowNumber;
- Object actualValue =
-
rowMeta.getValueMeta(valueIndex).convertToNormalStorageType(rowData[valueIndex]);
- assertEquals(
- expectedValue,
- actualValue,
- "Unexpected value at row " + rowNumber + " position " +
valueIndex);
- }
- }
- }
-
- assertEquals(2, rowNumber, "Incorrect output row number");
- }
-
- @Test
- @Disabled("This test needs to be reviewed")
- void testWithNormalStreams() throws HopException {
- doTest(false, false, false);
- }
-
- @Test
- @Disabled("This test needs to be reviewed")
- void testWithBinaryLookupStream() throws HopException {
- doTest(false, true, false);
- }
-
- @Test
- @Disabled("This test needs to be reviewed")
- void testWithBinaryDateStream() throws HopException {
- doTest(false, false, true);
- }
-
- @Test
- @Disabled("This test needs to be reviewed")
- void testWithBinaryStreams() throws HopException {
- doTest(false, false, true);
- }
-
- @Test
- void testMemoryPreservationWithNormalStreams() throws HopException {
- doTest(true, false, false);
- }
-
- @Test
- void testMemoryPreservationWithBinaryLookupStream() throws HopException {
- doTest(true, true, false);
- }
-
- @Test
- void testMemoryPreservationWithBinaryDateStream() throws HopException {
- doTest(true, false, true);
- }
-
- @Test
- void testMemoryPreservationWithBinaryStreams() throws HopException {
- doTest(true, false, true);
- }
-}
diff --git
a/plugins/transforms/streamlookup/src/test/resources/stream-lookup.xml
b/plugins/transforms/streamlookup/src/test/resources/stream-lookup.xml
new file mode 100644
index 0000000000..ee2640a3c0
--- /dev/null
+++ b/plugins/transforms/streamlookup/src/test/resources/stream-lookup.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<transform>
+ <from>Countries</from>
+ <input_sorted>Y</input_sorted>
+ <preserve_memory>Y</preserve_memory>
+ <sorted_list>Y</sorted_list>
+ <integer_pair>Y</integer_pair>
+ <lookup>
+ <key>
+ <name>keyStream1</name>
+ <field>keyLookup1</field>
+ </key>
+ <key>
+ <name>keyStream2</name>
+ <field>keyLookup2</field>
+ </key>
+ <value>
+ <name>returnValue1</name>
+ <rename>value1</rename>
+ <default/>
+ <type>String</type>
+ </value>
+ <value>
+ <name>returnValue2</name>
+ <rename>value2</rename>
+ <default>123</default>
+ <type>Integer</type>
+ </value>
+ <value>
+ <name>returnValue3</name>
+ <rename>value3</rename>
+ <default>true</default>
+ <type>Boolean</type>
+ </value>
+ </lookup>
+</transform>
