This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 6ee4d01945 EDI to XML escapes every data element through commons-text,
and the escaping has no tests, fixes #9465 (#9467)
6ee4d01945 is described below
commit 6ee4d01945457c83c94d2aae3909e99d7e8fb8d0
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Sat Nov 14 13:21:10 2026 +0100
EDI to XML escapes every data element through commons-text, and the
escaping has no tests, fixes #9465 (#9467)
---
.../transforms/0052-edi-to-xml-escaping-verify.hpl | 279 ++++++++++++++++++++
.../datasets/golden-edi-to-xml-escaping.csv | 160 +++++++++++
.../transforms/files/sample-escaping.edi | 10 +
.../transforms/main-0052-edi-to-xml.hwf | 3 +
.../dataset/golden-edi-to-xml-escaping.json | 72 +++++
.../0052-edi-to-xml-escaping-verify UNIT.json | 63 +++++
.../FastSimpleGenericEdifactDirectXMLParser.java | 29 +-
.../pipeline/transforms/edi2xml/Edi2XmlTest.java | 154 +++++++++++
.../transforms/edi2xml/EdifactToXmlParserTest.java | 293 +++++++++++++++++++++
9 files changed, 1061 insertions(+), 2 deletions(-)
diff --git a/integration-tests/transforms/0052-edi-to-xml-escaping-verify.hpl
b/integration-tests/transforms/0052-edi-to-xml-escaping-verify.hpl
new file mode 100644
index 0000000000..03c947fc4a
--- /dev/null
+++ b/integration-tests/transforms/0052-edi-to-xml-escaping-verify.hpl
@@ -0,0 +1,279 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0052-edi-to-xml-escaping-verify</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2022/11/25 13:51:18.060</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2022/11/25 13:51:18.060</modified_date>
+ </info>
+ <notepads>
+ <notepad>
+ <backgroundcolorblue>210</backgroundcolorblue>
+ <backgroundcolorgreen>136</backgroundcolorgreen>
+ <backgroundcolorred>15</backgroundcolorred>
+ <bordercolorblue>250</bordercolorblue>
+ <bordercolorgreen>231</bordercolorgreen>
+ <bordercolorred>200</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>250</fontcolorblue>
+ <fontcolorgreen>231</fontcolorgreen>
+ <fontcolorred>200</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontname>Noto Sans</fontname>
+ <fontsize>10</fontsize>
+ <height>75</height>
+ <xloc>96</xloc>
+ <yloc>192</yloc>
+ <note>Sample taken from the EDIFACT wikipedia page:
+
+https://en.wikipedia.org/wiki/EDIFACT
+
+</note>
+ <width>216</width>
+ </notepad>
+ </notepads>
+ <order>
+ <hop>
+ <from>sample.edi</from>
+ <to>EDI to XML</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>EDI to XML</from>
+ <to>flatten XML</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>flatten XML</from>
+ <to>limit</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>limit</from>
+ <to>Verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>EDI to XML</name>
+ <type>TypeExitEdi2XmlTransform</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <inputfield>edi</inputfield>
+ <outputfield>xml</outputfield>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>608</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>flatten XML</name>
+ <type>XMLInputStream</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <sourceFromInput>Y</sourceFromInput>
+ <sourceFieldName>xml</sourceFieldName>
+ <filename/>
+ <addResultFile>N</addResultFile>
+ <nrRowsToSkip>0</nrRowsToSkip>
+ <rowLimit>0</rowLimit>
+ <defaultStringLen>1024</defaultStringLen>
+ <encoding>UTF-8</encoding>
+ <enableNamespaces>N</enableNamespaces>
+ <enableTrim>Y</enableTrim>
+ <includeFilenameField>N</includeFilenameField>
+ <filenameField>xml_filename</filenameField>
+ <includeRowNumberField>N</includeRowNumberField>
+ <rowNumberField>xml_row_number</rowNumberField>
+ <includeDataTypeNumericField>N</includeDataTypeNumericField>
+ <dataTypeNumericField>xml_data_type_numeric</dataTypeNumericField>
+ <includeDataTypeDescriptionField>Y</includeDataTypeDescriptionField>
+
<dataTypeDescriptionField>xml_data_type_description</dataTypeDescriptionField>
+ <includeXmlLocationLineField>N</includeXmlLocationLineField>
+ <xmlLocationLineField>xml_location_line</xmlLocationLineField>
+ <includeXmlLocationColumnField>N</includeXmlLocationColumnField>
+ <xmlLocationColumnField>xml_location_column</xmlLocationColumnField>
+ <includeXmlElementIDField>Y</includeXmlElementIDField>
+ <xmlElementIDField>xml_element_id</xmlElementIDField>
+ <includeXmlParentElementIDField>Y</includeXmlParentElementIDField>
+ <xmlParentElementIDField>xml_parent_element_id</xmlParentElementIDField>
+ <includeXmlElementLevelField>Y</includeXmlElementLevelField>
+ <xmlElementLevelField>xml_element_level</xmlElementLevelField>
+ <includeXmlPathField>Y</includeXmlPathField>
+ <xmlPathField>xml_path</xmlPathField>
+ <includeXmlParentPathField>Y</includeXmlParentPathField>
+ <xmlParentPathField>xml_parent_path</xmlParentPathField>
+ <includeXmlDataNameField>Y</includeXmlDataNameField>
+ <xmlDataNameField>xml_data_name</xmlDataNameField>
+ <includeXmlDataValueField>Y</includeXmlDataValueField>
+ <xmlDataValueField>xml_data_value</xmlDataValueField>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>limit</name>
+ <type>SelectValues</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>xml_data_type_description</name>
+ </field>
+ <field>
+ <name>xml_element_id</name>
+ </field>
+ <field>
+ <name>xml_parent_element_id</name>
+ </field>
+ <field>
+ <name>xml_element_level</name>
+ </field>
+ <field>
+ <name>xml_path</name>
+ </field>
+ <field>
+ <name>xml_parent_path</name>
+ </field>
+ <field>
+ <name>xml_data_name</name>
+ </field>
+ <field>
+ <name>xml_data_value</name>
+ </field>
+ <select_unspecified>N</select_unspecified>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>464</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>sample.edi</name>
+ <type>LoadFileInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <addresultfile>Y</addresultfile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsIgnoreMissingPath>N</IsIgnoreMissingPath>
+ <rownum_field/>
+ <encoding>UTF-8</encoding>
+ <file>
+ <name>${PROJECT_HOME}/files/sample-escaping.edi</name>
+ <filemask/>
+ <exclude_filemask/>
+ <file_required>Y</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <fields>
+ <field>
+ <name>edi</name>
+ <element_type>content</element_type>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <IsInFields>N</IsInFields>
+ <DynamicFilenameField/>
+ <shortFileFieldName/>
+ <pathFieldName/>
+ <hiddenFieldName/>
+ <lastModificationTimeFieldName/>
+ <uriNameFieldName/>
+ <rootUriNameFieldName/>
+ <extensionFieldName/>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-edi-to-xml-escaping.csv
b/integration-tests/transforms/datasets/golden-edi-to-xml-escaping.csv
new file mode 100644
index 0000000000..cc15815e4f
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-edi-to-xml-escaping.csv
@@ -0,0 +1,160 @@
+xml_data_type_description,xml_element_id,xml_parent_element_id,xml_element_level,xml_path,xml_parent_path,xml_data_name,xml_data_value
+START_DOCUMENT,0,,0,,,,
+START_ELEMENT,1,0,1,/edifact,,edifact,
+START_ELEMENT,2,1,2,/edifact/UNB,/edifact,UNB,
+START_ELEMENT,3,2,3,/edifact/UNB/element,/edifact/UNB,element,
+START_ELEMENT,4,3,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+CHARACTERS,4,3,4,/edifact/UNB/element/value,/edifact/UNB/element,value,IATB
+END_ELEMENT,4,3,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+START_ELEMENT,5,3,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+CHARACTERS,5,3,4,/edifact/UNB/element/value,/edifact/UNB/element,value,1
+END_ELEMENT,5,3,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+END_ELEMENT,3,2,3,/edifact/UNB/element,/edifact/UNB,element,
+START_ELEMENT,6,2,3,/edifact/UNB/element,/edifact/UNB,element,
+START_ELEMENT,7,6,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+CHARACTERS,7,6,4,/edifact/UNB/element/value,/edifact/UNB/element,value,6XPPC
+END_ELEMENT,7,6,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+START_ELEMENT,8,6,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+CHARACTERS,8,6,4,/edifact/UNB/element/value,/edifact/UNB/element,value,ZZ
+END_ELEMENT,8,6,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+END_ELEMENT,6,2,3,/edifact/UNB/element,/edifact/UNB,element,
+START_ELEMENT,9,2,3,/edifact/UNB/element,/edifact/UNB,element,
+START_ELEMENT,10,9,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+CHARACTERS,10,9,4,/edifact/UNB/element/value,/edifact/UNB/element,value,LHPPC
+END_ELEMENT,10,9,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+START_ELEMENT,11,9,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+CHARACTERS,11,9,4,/edifact/UNB/element/value,/edifact/UNB/element,value,ZZ
+END_ELEMENT,11,9,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+END_ELEMENT,9,2,3,/edifact/UNB/element,/edifact/UNB,element,
+START_ELEMENT,12,2,3,/edifact/UNB/element,/edifact/UNB,element,
+START_ELEMENT,13,12,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+CHARACTERS,13,12,4,/edifact/UNB/element/value,/edifact/UNB/element,value,940101
+END_ELEMENT,13,12,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+START_ELEMENT,14,12,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+CHARACTERS,14,12,4,/edifact/UNB/element/value,/edifact/UNB/element,value,0950
+END_ELEMENT,14,12,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+END_ELEMENT,12,2,3,/edifact/UNB/element,/edifact/UNB,element,
+START_ELEMENT,15,2,3,/edifact/UNB/element,/edifact/UNB,element,
+START_ELEMENT,16,15,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+CHARACTERS,16,15,4,/edifact/UNB/element/value,/edifact/UNB/element,value,1
+END_ELEMENT,16,15,4,/edifact/UNB/element/value,/edifact/UNB/element,value,
+END_ELEMENT,15,2,3,/edifact/UNB/element,/edifact/UNB,element,
+END_ELEMENT,2,1,2,/edifact/UNB,/edifact,UNB,
+START_ELEMENT,17,1,2,/edifact/UNH,/edifact,UNH,
+START_ELEMENT,18,17,3,/edifact/UNH/element,/edifact/UNH,element,
+START_ELEMENT,19,18,4,/edifact/UNH/element/value,/edifact/UNH/element,value,
+CHARACTERS,19,18,4,/edifact/UNH/element/value,/edifact/UNH/element,value,1
+END_ELEMENT,19,18,4,/edifact/UNH/element/value,/edifact/UNH/element,value,
+END_ELEMENT,18,17,3,/edifact/UNH/element,/edifact/UNH,element,
+START_ELEMENT,20,17,3,/edifact/UNH/element,/edifact/UNH,element,
+START_ELEMENT,21,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,
+CHARACTERS,21,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,PAORES
+END_ELEMENT,21,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,
+START_ELEMENT,22,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,
+CHARACTERS,22,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,93
+END_ELEMENT,22,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,
+START_ELEMENT,23,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,
+CHARACTERS,23,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,1
+END_ELEMENT,23,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,
+START_ELEMENT,24,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,
+CHARACTERS,24,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,IA
+END_ELEMENT,24,20,4,/edifact/UNH/element/value,/edifact/UNH/element,value,
+END_ELEMENT,20,17,3,/edifact/UNH/element,/edifact/UNH,element,
+END_ELEMENT,17,1,2,/edifact/UNH,/edifact,UNH,
+START_ELEMENT,25,1,2,/edifact/MSG,/edifact,MSG,
+START_ELEMENT,26,25,3,/edifact/MSG/element,/edifact/MSG,element,
+START_ELEMENT,27,26,4,/edifact/MSG/element/value,/edifact/MSG/element,value,
+CHARACTERS,27,26,4,/edifact/MSG/element/value,/edifact/MSG/element,value,1
+END_ELEMENT,27,26,4,/edifact/MSG/element/value,/edifact/MSG/element,value,
+START_ELEMENT,28,26,4,/edifact/MSG/element/value,/edifact/MSG/element,value,
+CHARACTERS,28,26,4,/edifact/MSG/element/value,/edifact/MSG/element,value,45
+END_ELEMENT,28,26,4,/edifact/MSG/element/value,/edifact/MSG/element,value,
+END_ELEMENT,26,25,3,/edifact/MSG/element,/edifact/MSG,element,
+END_ELEMENT,25,1,2,/edifact/MSG,/edifact,MSG,
+START_ELEMENT,29,1,2,/edifact/IFT,/edifact,IFT,
+START_ELEMENT,30,29,3,/edifact/IFT/element,/edifact/IFT,element,
+START_ELEMENT,31,30,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+CHARACTERS,31,30,4,/edifact/IFT/element/value,/edifact/IFT/element,value,3
+END_ELEMENT,31,30,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+END_ELEMENT,30,29,3,/edifact/IFT/element,/edifact/IFT,element,
+START_ELEMENT,32,29,3,/edifact/IFT/element,/edifact/IFT,element,
+START_ELEMENT,33,32,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+CHARACTERS,33,32,4,/edifact/IFT/element/value,/edifact/IFT/element,value,ACME
& SONS <LTD> AVAILABILITY
+END_ELEMENT,33,32,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+END_ELEMENT,32,29,3,/edifact/IFT/element,/edifact/IFT,element,
+END_ELEMENT,29,1,2,/edifact/IFT,/edifact,IFT,
+START_ELEMENT,34,1,2,/edifact/IFT,/edifact,IFT,
+START_ELEMENT,35,34,3,/edifact/IFT/element,/edifact/IFT,element,
+START_ELEMENT,36,35,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+CHARACTERS,36,35,4,/edifact/IFT/element/value,/edifact/IFT/element,value,3
+END_ELEMENT,36,35,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+END_ELEMENT,35,34,3,/edifact/IFT/element,/edifact/IFT,element,
+START_ELEMENT,37,34,3,/edifact/IFT/element,/edifact/IFT,element,
+START_ELEMENT,38,37,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+CHARACTERS,38,37,4,/edifact/IFT/element/value,/edifact/IFT/element,value,"QUOTED
""TEXT"" AND AN APOSTROPHE ' HERE"
+END_ELEMENT,38,37,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+END_ELEMENT,37,34,3,/edifact/IFT/element,/edifact/IFT,element,
+END_ELEMENT,34,1,2,/edifact/IFT,/edifact,IFT,
+START_ELEMENT,39,1,2,/edifact/IFT,/edifact,IFT,
+START_ELEMENT,40,39,3,/edifact/IFT/element,/edifact/IFT,element,
+START_ELEMENT,41,40,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+CHARACTERS,41,40,4,/edifact/IFT/element/value,/edifact/IFT/element,value,3
+END_ELEMENT,41,40,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+END_ELEMENT,40,39,3,/edifact/IFT/element,/edifact/IFT,element,
+START_ELEMENT,42,39,3,/edifact/IFT/element,/edifact/IFT,element,
+START_ELEMENT,43,42,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+CHARACTERS,43,42,4,/edifact/IFT/element/value,/edifact/IFT/element,value,RELEASED
+ PLUS : COLON ? QUESTION MARK
+END_ELEMENT,43,42,4,/edifact/IFT/element/value,/edifact/IFT/element,value,
+END_ELEMENT,42,39,3,/edifact/IFT/element,/edifact/IFT,element,
+END_ELEMENT,39,1,2,/edifact/IFT,/edifact,IFT,
+START_ELEMENT,44,1,2,/edifact/NAD,/edifact,NAD,
+START_ELEMENT,45,44,3,/edifact/NAD/element,/edifact/NAD,element,
+START_ELEMENT,46,45,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+CHARACTERS,46,45,4,/edifact/NAD/element/value,/edifact/NAD/element,value,BY
+END_ELEMENT,46,45,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+END_ELEMENT,45,44,3,/edifact/NAD/element,/edifact/NAD,element,
+START_ELEMENT,47,44,3,/edifact/NAD/element,/edifact/NAD,element,
+START_ELEMENT,48,47,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+END_ELEMENT,48,47,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+END_ELEMENT,47,44,3,/edifact/NAD/element,/edifact/NAD,element,
+START_ELEMENT,49,44,3,/edifact/NAD/element,/edifact/NAD,element,
+START_ELEMENT,50,49,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+END_ELEMENT,50,49,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+END_ELEMENT,49,44,3,/edifact/NAD/element,/edifact/NAD,element,
+START_ELEMENT,51,44,3,/edifact/NAD/element,/edifact/NAD,element,
+START_ELEMENT,52,51,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+CHARACTERS,52,51,4,/edifact/NAD/element/value,/edifact/NAD/element,value,O'BRIEN
& CO
+END_ELEMENT,52,51,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+START_ELEMENT,53,51,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+END_ELEMENT,53,51,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+START_ELEMENT,54,51,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+CHARACTERS,54,51,4,/edifact/NAD/element/value,/edifact/NAD/element,value,CAFÉ
+END_ELEMENT,54,51,4,/edifact/NAD/element/value,/edifact/NAD/element,value,
+END_ELEMENT,51,44,3,/edifact/NAD/element,/edifact/NAD,element,
+END_ELEMENT,44,1,2,/edifact/NAD,/edifact,NAD,
+START_ELEMENT,55,1,2,/edifact/UNT,/edifact,UNT,
+START_ELEMENT,56,55,3,/edifact/UNT/element,/edifact/UNT,element,
+START_ELEMENT,57,56,4,/edifact/UNT/element/value,/edifact/UNT/element,value,
+CHARACTERS,57,56,4,/edifact/UNT/element/value,/edifact/UNT/element,value,7
+END_ELEMENT,57,56,4,/edifact/UNT/element/value,/edifact/UNT/element,value,
+END_ELEMENT,56,55,3,/edifact/UNT/element,/edifact/UNT,element,
+START_ELEMENT,58,55,3,/edifact/UNT/element,/edifact/UNT,element,
+START_ELEMENT,59,58,4,/edifact/UNT/element/value,/edifact/UNT/element,value,
+CHARACTERS,59,58,4,/edifact/UNT/element/value,/edifact/UNT/element,value,1
+END_ELEMENT,59,58,4,/edifact/UNT/element/value,/edifact/UNT/element,value,
+END_ELEMENT,58,55,3,/edifact/UNT/element,/edifact/UNT,element,
+END_ELEMENT,55,1,2,/edifact/UNT,/edifact,UNT,
+START_ELEMENT,60,1,2,/edifact/UNZ,/edifact,UNZ,
+START_ELEMENT,61,60,3,/edifact/UNZ/element,/edifact/UNZ,element,
+START_ELEMENT,62,61,4,/edifact/UNZ/element/value,/edifact/UNZ/element,value,
+CHARACTERS,62,61,4,/edifact/UNZ/element/value,/edifact/UNZ/element,value,1
+END_ELEMENT,62,61,4,/edifact/UNZ/element/value,/edifact/UNZ/element,value,
+END_ELEMENT,61,60,3,/edifact/UNZ/element,/edifact/UNZ,element,
+START_ELEMENT,63,60,3,/edifact/UNZ/element,/edifact/UNZ,element,
+START_ELEMENT,64,63,4,/edifact/UNZ/element/value,/edifact/UNZ/element,value,
+CHARACTERS,64,63,4,/edifact/UNZ/element/value,/edifact/UNZ/element,value,1
+END_ELEMENT,64,63,4,/edifact/UNZ/element/value,/edifact/UNZ/element,value,
+END_ELEMENT,63,60,3,/edifact/UNZ/element,/edifact/UNZ,element,
+END_ELEMENT,60,1,2,/edifact/UNZ,/edifact,UNZ,
+END_ELEMENT,1,0,1,/edifact,,edifact,
+END_DOCUMENT,0,,0,,,,
diff --git a/integration-tests/transforms/files/sample-escaping.edi
b/integration-tests/transforms/files/sample-escaping.edi
new file mode 100644
index 0000000000..6a24d55d2d
--- /dev/null
+++ b/integration-tests/transforms/files/sample-escaping.edi
@@ -0,0 +1,10 @@
+UNA:+.? '
+UNB+IATB:1+6XPPC:ZZ+LHPPC:ZZ+940101:0950+1'
+UNH+1+PAORES:93:1:IA'
+MSG+1:45'
+IFT+3+ACME & SONS <LTD> AVAILABILITY'
+IFT+3+QUOTED "TEXT" AND AN APOSTROPHE ?' HERE'
+IFT+3+RELEASED ?+ PLUS ?: COLON ?? QUESTION MARK'
+NAD+BY+++O?'BRIEN & CO::CAFÉ'
+UNT+7+1'
+UNZ+1+1'
diff --git a/integration-tests/transforms/main-0052-edi-to-xml.hwf
b/integration-tests/transforms/main-0052-edi-to-xml.hwf
index 19f760aa3d..5bc0ced835 100644
--- a/integration-tests/transforms/main-0052-edi-to-xml.hwf
+++ b/integration-tests/transforms/main-0052-edi-to-xml.hwf
@@ -57,6 +57,9 @@ limitations under the License.
<test_name>
<name>0052-edi-to-xml-verify UNIT</name>
</test_name>
+ <test_name>
+ <name>0052-edi-to-xml-escaping-verify UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>336</xloc>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-edi-to-xml-escaping.json
b/integration-tests/transforms/metadata/dataset/golden-edi-to-xml-escaping.json
new file mode 100644
index 0000000000..262ba12cee
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-edi-to-xml-escaping.json
@@ -0,0 +1,72 @@
+{
+ "base_filename": "golden-edi-to-xml-escaping.csv",
+ "name": "golden-edi-to-xml-escaping",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": 25,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "xml_data_type_description",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": 10,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "xml_element_id",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": 10,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "xml_parent_element_id",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": 10,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "xml_element_level",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": 1024,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "xml_path",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": 1024,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "xml_parent_path",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": 1024,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "xml_data_name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": 1024,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "xml_data_value",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0052-edi-to-xml-escaping-verify
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0052-edi-to-xml-escaping-verify
UNIT.json
new file mode 100644
index 0000000000..48e2d2f791
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0052-edi-to-xml-escaping-verify
UNIT.json
@@ -0,0 +1,63 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "xml_data_type_description",
+ "data_set_field": "xml_data_type_description"
+ },
+ {
+ "transform_field": "xml_element_id",
+ "data_set_field": "xml_element_id"
+ },
+ {
+ "transform_field": "xml_parent_element_id",
+ "data_set_field": "xml_parent_element_id"
+ },
+ {
+ "transform_field": "xml_element_level",
+ "data_set_field": "xml_element_level"
+ },
+ {
+ "transform_field": "xml_path",
+ "data_set_field": "xml_path"
+ },
+ {
+ "transform_field": "xml_parent_path",
+ "data_set_field": "xml_parent_path"
+ },
+ {
+ "transform_field": "xml_data_name",
+ "data_set_field": "xml_data_name"
+ },
+ {
+ "transform_field": "xml_data_value",
+ "data_set_field": "xml_data_value"
+ }
+ ],
+ "field_order": [
+ "xml_data_type_description",
+ "xml_element_id",
+ "xml_parent_element_id",
+ "xml_element_level",
+ "xml_path",
+ "xml_parent_path",
+ "xml_data_name",
+ "xml_data_value"
+ ],
+ "data_set_name": "golden-edi-to-xml-escaping",
+ "transform_name": "Verify"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0052-edi-to-xml-escaping-verify UNIT",
+ "description": "",
+ "persist_filename": "",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0052-edi-to-xml-escaping-verify.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
diff --git
a/plugins/transforms/edi2xml/src/main/java/org/apache/hop/pipeline/transforms/edi2xml/grammar/FastSimpleGenericEdifactDirectXMLParser.java
b/plugins/transforms/edi2xml/src/main/java/org/apache/hop/pipeline/transforms/edi2xml/grammar/FastSimpleGenericEdifactDirectXMLParser.java
index 8dde30c25d..b7121610b4 100644
---
a/plugins/transforms/edi2xml/src/main/java/org/apache/hop/pipeline/transforms/edi2xml/grammar/FastSimpleGenericEdifactDirectXMLParser.java
+++
b/plugins/transforms/edi2xml/src/main/java/org/apache/hop/pipeline/transforms/edi2xml/grammar/FastSimpleGenericEdifactDirectXMLParser.java
@@ -149,8 +149,33 @@ public class FastSimpleGenericEdifactDirectXMLParser
extends Parser {
txt = txt.replace("??", "?");
}
- // enocde XML entities
- return StringEscapeUtils.escapeXml10(txt);
+ // encode XML entities: almost no EDIFACT value has a character that needs
it
+ return needsEscaping(txt) ? StringEscapeUtils.escapeXml10(txt) : txt;
+ }
+
+ /**
+ * Whether a text has a character that {@link
StringEscapeUtils#escapeXml10(String)} changes: one
+ * of the five XML characters, a control character, a surrogate or a
character XML 1.0 does not
+ * allow. Escaping a text without any of them gives the text itself, so the
call can be skipped.
+ *
+ * <p>The test is on the safe side: a character it flags is handed to
escapeXml10, which decides.
+ */
+ public static boolean needsEscaping(String txt) {
+ for (int i = 0; i < txt.length(); i++) {
+ char c = txt.charAt(i);
+ if (c < 0x20) {
+ if (c != '\t' && c != '\n' && c != '\r') {
+ return true;
+ }
+ } else if (c < 0x7f) {
+ if (c == '&' || c == '<' || c == '>' || c == '"' || c == '\'') {
+ return true;
+ }
+ } else if (c < 0xa0 || (c >= 0xd800 && c <= 0xdfff) || c >= 0xfffe) {
+ return true;
+ }
+ }
+ return false;
}
// assume about 8k for an edifact message
diff --git
a/plugins/transforms/edi2xml/src/test/java/org/apache/hop/pipeline/transforms/edi2xml/Edi2XmlTest.java
b/plugins/transforms/edi2xml/src/test/java/org/apache/hop/pipeline/transforms/edi2xml/Edi2XmlTest.java
new file mode 100644
index 0000000000..7470a134d3
--- /dev/null
+++
b/plugins/transforms/edi2xml/src/test/java/org/apache/hop/pipeline/transforms/edi2xml/Edi2XmlTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.edi2xml;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** The EDI to XML transform, row by row. */
+class Edi2XmlTest {
+
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ private TransformMockHelper<Edi2XmlMeta, Edi2XmlData> mockHelper;
+
+ @BeforeEach
+ void setUp() {
+ mockHelper = new TransformMockHelper<>("EDI to XML", Edi2XmlMeta.class,
Edi2XmlData.class);
+ when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(mockHelper.iLogChannel);
+ when(mockHelper.pipeline.isRunning()).thenReturn(true);
+ when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(false);
+ }
+
+ @AfterEach
+ void tearDown() {
+ mockHelper.cleanUp();
+ }
+
+ private Edi2Xml transform(String outputField, BlockingRowSet output,
Object[]... rows) {
+ Edi2XmlMeta meta = new Edi2XmlMeta();
+ meta.setInputField("edi");
+ meta.setOutputField(outputField);
+ when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("id"));
+ rowMeta.addValueMeta(new ValueMetaString("edi"));
+ BlockingRowSet input = new BlockingRowSet(rows.length + 1);
+ input.setThreadNameFromToCopy("previous", 0, "EDI to XML", 0);
+ input.setRowMeta(rowMeta);
+ for (Object[] row : rows) {
+ input.putRow(rowMeta, row);
+ }
+ input.setDone();
+ output.setThreadNameFromToCopy("EDI to XML", 0, "next", 0);
+
+ Edi2Xml transform =
+ new Edi2Xml(
+ mockHelper.transformMeta,
+ meta,
+ new Edi2XmlData(),
+ 0,
+ mockHelper.pipelineMeta,
+ mockHelper.pipeline);
+ transform.addRowSetToInputRowSets(input);
+ transform.addRowSetToOutputRowSets(output);
+ return transform;
+ }
+
+ @Test
+ void everyRowGetsTheXmlOfItsOwnMessageInTheOutputField() throws Exception {
+ BlockingRowSet output = new BlockingRowSet(10);
+ Edi2Xml transform =
+ transform(
+ "xml",
+ output,
+ new Object[] {1L, "UNH+1+ACME & Sons'"},
+ new Object[] {2L, "BGM+220+O?'Brien'"},
+ new Object[] {3L, null});
+ assertTrue(transform.init());
+
+ assertTrue(transform.processRow());
+ assertTrue(transform.processRow());
+ assertTrue(transform.processRow());
+ assertFalse(transform.processRow());
+
+ Object[] first = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(1L, first[0]);
+ assertEquals("UNH+1+ACME & Sons'", first[1]);
+ assertTrue(((String) first[2]).contains("<value>ACME & Sons</value>"),
(String) first[2]);
+
+ Object[] second = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(2L, second[0]);
+ String secondXml = (String) second[2];
+ assertTrue(secondXml.contains("<BGM>"), secondXml);
+ assertTrue(secondXml.contains("<value>O'Brien</value>"), secondXml);
+ // nothing of the first message is left in the second
+ assertFalse(secondXml.contains("UNH"), secondXml);
+
+ // an empty input is an empty document, not an error
+ Object[] third = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(3L, third[0]);
+ assertFalse(((String) third[2]).contains("<value>"));
+ assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS));
+ }
+
+ @Test
+ void withoutAnOutputFieldTheXmlReplacesTheInput() throws Exception {
+ BlockingRowSet output = new BlockingRowSet(10);
+ Edi2Xml transform = transform("", output, new Object[] {1L, "UNH+1+X'"});
+ assertTrue(transform.init());
+
+ assertTrue(transform.processRow());
+
+ Object[] row = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(1L, row[0]);
+ assertTrue(((String) row[1]).contains("<value>X</value>"), (String)
row[1]);
+ }
+
+ @Test
+ void aMessageThatIsNotEdifactFailsTheTransform() {
+ BlockingRowSet output = new BlockingRowSet(10);
+ Edi2Xml transform = transform("xml", output, new Object[] {1L, "this is
not edifact"});
+ assertTrue(transform.init());
+
+ assertThrows(HopException.class, transform::processRow);
+ }
+}
diff --git
a/plugins/transforms/edi2xml/src/test/java/org/apache/hop/pipeline/transforms/edi2xml/EdifactToXmlParserTest.java
b/plugins/transforms/edi2xml/src/test/java/org/apache/hop/pipeline/transforms/edi2xml/EdifactToXmlParserTest.java
new file mode 100644
index 0000000000..bed5638409
--- /dev/null
+++
b/plugins/transforms/edi2xml/src/test/java/org/apache/hop/pipeline/transforms/edi2xml/EdifactToXmlParserTest.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.edi2xml;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.xml.parsers.DocumentBuilderFactory;
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.apache.commons.text.StringEscapeUtils;
+import
org.apache.hop.pipeline.transforms.edi2xml.grammar.FastSimpleGenericEdifactDirectXMLLexer;
+import
org.apache.hop.pipeline.transforms.edi2xml.grammar.FastSimpleGenericEdifactDirectXMLParser;
+import org.junit.jupiter.api.Test;
+import org.w3c.dom.Document;
+
+/**
+ * The XML the EDIFACT parser writes: which characters of a value are escaped,
what happens to the
+ * release character, and the layout of segments, elements and values.
+ */
+class EdifactToXmlParserTest {
+
+ private static final Pattern VALUE = Pattern.compile("<value>(.*?)</value>",
Pattern.DOTALL);
+
+ // not as unicode escapes in the source: the compiler reads those before it
reads the string
+ private static final char CONTROL_1 = (char) 0x01;
+ private static final char DELETE = (char) 0x7f;
+ private static final char HIGH_SURROGATE = (char) 0xd83d;
+ private static final char LOW_SURROGATE = (char) 0xde00;
+
+ /** Runs the lexer and parser over a message, as the transform does for the
first row. */
+ private static String toXml(String edi) throws Exception {
+ FastSimpleGenericEdifactDirectXMLLexer lexer =
+ new FastSimpleGenericEdifactDirectXMLLexer(new ANTLRStringStream(edi));
+ FastSimpleGenericEdifactDirectXMLParser parser =
+ new FastSimpleGenericEdifactDirectXMLParser(new
CommonTokenStream(lexer));
+ parser.edifact();
+ return parser.buf.toString();
+ }
+
+ /** The text between the value tags, as written: still escaped. */
+ private static List<String> values(String xml) {
+ List<String> values = new ArrayList<>();
+ Matcher matcher = VALUE.matcher(xml);
+ while (matcher.find()) {
+ values.add(matcher.group(1));
+ }
+ return values;
+ }
+
+ private static Document parse(String xml) throws Exception {
+ return DocumentBuilderFactory.newInstance()
+ .newDocumentBuilder()
+ .parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)));
+ }
+
+ private static String sanitize(String text) {
+ FastSimpleGenericEdifactDirectXMLParser parser =
+ new FastSimpleGenericEdifactDirectXMLParser(
+ new CommonTokenStream(
+ new FastSimpleGenericEdifactDirectXMLLexer(new
ANTLRStringStream(""))));
+ return parser.sanitizeText(text);
+ }
+
+ // --- sanitizeText: one value ---
+
+ @Test
+ void aValueWithoutSpecialCharactersIsLeftAsItIs() {
+ assertEquals("ORDERS", sanitize("ORDERS"));
+ assertEquals("12 MAIN STREET", sanitize("12 MAIN STREET"));
+ assertEquals("14.58", sanitize("14.58"));
+ assertEquals("", sanitize(""));
+ assertEquals("tab\there", sanitize("tab\there"));
+ }
+
+ @Test
+ void theFiveXmlCharactersAreEscaped() {
+ assertEquals("ACME & Sons <Ltd>", sanitize("ACME & Sons <Ltd>"));
+ assertEquals("say "hi" it's", sanitize("say \"hi\" it's"));
+ assertEquals("&&", sanitize("&&"));
+ assertEquals("&amp;", sanitize("&"));
+ }
+
+ @Test
+ void theReleaseCharacterIsResolvedBeforeEscaping() {
+ assertEquals("a + b : c ' d ? e", sanitize("a ?+ b ?: c ?' d ?? e"));
+ // a question mark that releases nothing stays
+ assertEquals("?", sanitize("?"));
+ assertEquals("a?b", sanitize("a?b"));
+ // a released character that needs escaping is escaped
+ assertEquals("O'Brien & Co", sanitize("O?'Brien & Co"));
+ }
+
+ @Test
+ void textOutsideAsciiIsKept() {
+ assertEquals("café € ñ", sanitize("café € ñ"));
+ String emoji = "ok " + HIGH_SURROGATE + LOW_SURROGATE;
+ assertEquals(emoji, sanitize(emoji));
+ }
+
+ @Test
+ void charactersXmlCannotHoldAreRemovedOrWrittenAsAReference() {
+ assertEquals("ctl x", sanitize("ctl " + CONTROL_1 + " x"));
+ assertEquals("del  x", sanitize("del " + DELETE + " x"));
+ // half of a surrogate pair is not a character
+ assertEquals("lone x", sanitize("lone " + HIGH_SURROGATE + " x"));
+ }
+
+ // --- the check that skips the escaping ---
+
+ /** For every character: when the check says nothing needs escaping,
escaping changes nothing. */
+ @Test
+ void aCharacterThatIsNotFlaggedIsNeverChangedByTheEscaping() {
+ int passed = 0;
+ for (int c = 0; c <= 0xffff; c++) {
+ String text = String.valueOf((char) c);
+ if (!FastSimpleGenericEdifactDirectXMLParser.needsEscaping(text)) {
+ passed++;
+ assertEquals(text, StringEscapeUtils.escapeXml10(text), "character " +
c);
+ }
+ }
+ // nearly every character passes: the check is not just always saying yes
+ assertTrue(passed > 60_000, "characters that pass: " + passed);
+ }
+
+ @Test
+ void theOutcomeIsThatOfTheEscapingForMixedTexts() {
+ List<String> texts =
+ List.of(
+ "ORDERS",
+ "ACME & Sons <Ltd>",
+ "say \"hi\" it's",
+ "café € ñ",
+ "ok " + HIGH_SURROGATE + LOW_SURROGATE,
+ "lone " + HIGH_SURROGATE + " x",
+ "ctl " + CONTROL_1 + " x",
+ "del " + DELETE + " x",
+ "tab\there\nand a line",
+ "");
+ for (String text : texts) {
+ assertEquals(StringEscapeUtils.escapeXml10(text), sanitize(text), text);
+ }
+ }
+
+ // --- whole messages ---
+
+ @Test
+ void aSmallMessageGivesThisXml() throws Exception {
+ String xml = toXml("UNH+1+ORDERS:D:96A:UN'\nBGM+220+128576'\nUNT+3+1'");
+
+ assertEquals(
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<edifact>\n"
+ + "\t<UNH>\n"
+ + "\t\t<element>\n"
+ + "\t\t\t<value>1</value>\n"
+ + "\t\t</element>\n"
+ + "\t\t<element>\n"
+ + "\t\t\t<value>ORDERS</value>\n"
+ + "\t\t\t<value>D</value>\n"
+ + "\t\t\t<value>96A</value>\n"
+ + "\t\t\t<value>UN</value>\n"
+ + "\t\t</element>\n"
+ + "\t</UNH>\n"
+ + "\t<BGM>\n"
+ + "\t\t<element>\n"
+ + "\t\t\t<value>220</value>\n"
+ + "\t\t</element>\n"
+ + "\t\t<element>\n"
+ + "\t\t\t<value>128576</value>\n"
+ + "\t\t</element>\n"
+ + "\t</BGM>\n"
+ + "\t<UNT>\n"
+ + "\t\t<element>\n"
+ + "\t\t\t<value>3</value>\n"
+ + "\t\t</element>\n"
+ + "\t\t<element>\n"
+ + "\t\t\t<value>1</value>\n"
+ + "\t\t</element>\n"
+ + "\t</UNT>\n"
+ + "</edifact>",
+ xml);
+ }
+
+ @Test
+ void valuesThatNeedEscapingGiveWellFormedXmlWithTheOriginalText() throws
Exception {
+ String xml =
+ toXml(
+ "UNH+1+PAORES:93:1:IA'\n"
+ + "IFT+3+ACME & Sons <Ltd> ?+ more'\n"
+ + "NAD+BY+++O?'Brien::\"x\"'\n"
+ + "UNT+4+1'");
+
+ assertEquals(
+ List.of(
+ "1",
+ "PAORES",
+ "93",
+ "1",
+ "IA",
+ "3",
+ "ACME & Sons <Ltd> + more",
+ "BY",
+ "",
+ "",
+ "O'Brien",
+ "",
+ ""x"",
+ "4",
+ "1"),
+ values(xml));
+
+ // a reader of the XML gets the text of the message back
+ Document document = parse(xml);
+ assertEquals(
+ "ACME & Sons <Ltd> + more",
+ document
+ .getElementsByTagName("IFT")
+ .item(0)
+ .getLastChild()
+ .getPreviousSibling()
+ .getTextContent()
+ .trim());
+ assertEquals(15, document.getElementsByTagName("value").getLength());
+ }
+
+ @Test
+ void emptyElementsAndComponentsKeepTheirPlace() throws Exception {
+ String xml = toXml("PDI++C:3+Y::3+F::1'");
+
+ assertEquals(List.of("", "C", "3", "Y", "", "3", "F", "", "1"),
values(xml));
+ }
+
+ /** The transform uses one parser for every row. */
+ @Test
+ void oneParserHandlesOneMessageAfterTheOther() throws Exception {
+ FastSimpleGenericEdifactDirectXMLLexer lexer =
+ new FastSimpleGenericEdifactDirectXMLLexer(new
ANTLRStringStream("UNH+1+A & B'"));
+ CommonTokenStream tokens = new CommonTokenStream(lexer);
+ FastSimpleGenericEdifactDirectXMLParser parser =
+ new FastSimpleGenericEdifactDirectXMLParser(tokens);
+ parser.edifact();
+ assertEquals(List.of("1", "A & B"), values(parser.buf.toString()));
+
+ lexer.setCharStream(new ANTLRStringStream("BGM+220+X'"));
+ tokens.setTokenSource(lexer);
+ parser.setTokenStream(tokens);
+ parser.edifact();
+
+ String second = parser.buf.toString();
+ assertEquals(List.of("220", "X"), values(second));
+ assertFalse(second.contains("UNH"));
+ }
+
+ @Test
+ void theSampleMessageOfThePluginGivesWellFormedXml() throws Exception {
+ String edi =
+ Files.readString(
+ Paths.get("src/main/samples/files/edi-to-xml.txt"),
StandardCharsets.UTF_8);
+
+ String xml = toXml(edi);
+
+ Document document = parse(xml);
+ assertEquals("edifact", document.getDocumentElement().getTagName());
+ assertTrue(values(xml).contains("NO MORE FLIGHTS"), xml);
+ assertEquals(1, document.getElementsByTagName("UNB").getLength());
+ assertEquals(1, document.getElementsByTagName("UNZ").getLength());
+ }
+}