This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 7c2d8e0452 HTML to text drops rows with empty HTML, can hang in
parallel mode, a… (#9454)
7c2d8e0452 is described below
commit 7c2d8e0452de6d5543eb6f6ceded93dca9ed5905
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Sun Nov 15 08:55:38 2026 +0100
HTML to text drops rows with empty HTML, can hang in parallel mode, a…
(#9454)
* HTML to text drops rows with empty HTML, can hang in parallel mode, and
creates a new parser for every row, fixes #9451
* fix remarks
* fix remarks
---
.../ROOT/pages/pipeline/transforms/html2text.adoc | 25 ++
integration-tests/transforms/0127-html2text.hpl | 212 ++++++++++++++++
.../datasets/golden-0127-html2text-copies.csv | 4 +
.../transforms/datasets/golden-0127-html2text.csv | 4 +
.../transforms/main-0127-html2text.hwf | 79 ++++++
.../dataset/golden-0127-html2text-copies.json | 24 ++
.../metadata/dataset/golden-0127-html2text.json | 24 ++
.../metadata/unit-test/0127-html2text UNIT.json | 49 ++++
.../pipeline/transforms/html2text/Html2Text.java | 150 +++--------
.../transforms/html2text/Html2TextData.java | 3 +
.../transforms/html2text/Html2TextDialog.java | 30 ---
.../transforms/html2text/Html2TextMeta.java | 3 -
.../transforms/html2text/HtmlConverter.java | 86 +++++++
.../html2text/messages/messages_en_US.properties | 2 -
.../html2text/messages/messages_pt_BR.properties | 1 -
.../src/main/samples/transforms/html2text.hpl | 3 -
.../transforms/html2text/Html2TextMetaTest.java | 2 +-
.../transforms/html2text/Html2TextTest.java | 277 +++++++++++++++++++++
.../transforms/html2text/HtmlConverterTest.java | 111 +++++++++
19 files changed, 928 insertions(+), 161 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/html2text.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/html2text.adoc
index 28546fc0ca..2b04f5e64b 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/html2text.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/html2text.adoc
@@ -23,3 +23,28 @@ under the License.
== Description
The HTML 2 Text Transform allows you to parse an HTML page and convert the
content to text
+
+For every row, the transform reads the HTML from a field and adds a field with
the text of that HTML, or, with *Clean HTML Only*, with the HTML cleaned of
everything the safelist does not allow.
+
+A row whose HTML field is empty, holds only white space or is null passes on
with an empty text.
+
+NOTE: Before Hop 2.21, rows with empty HTML were dropped: they did not reach
the output.
+
+== Options
+
+[options="header"]
+|===
+|Option|Description
+|Transform name|Name of the transform, unique in the pipeline.
+|Html text field|The field that holds the HTML.
+|Output text field|The name of the new field with the text, or the cleaned
HTML.
+|Clean HTML Only|Keep the HTML and remove the elements and attributes that the
safelist does not allow, instead of turning the HTML into text.
+|Safelist|With *Clean HTML Only*: what the cleaned HTML may keep. *None* keeps
only the text, *Simple text* adds bold, italic and underline, *Basic* adds
links, lists, quotes and paragraphs, *Basic with images* adds images, and
*Relaxed* adds headings, tables, divisions and other structural elements.
+|Normalize Text|For text: replace runs of white space, line breaks included,
with a single space. Without it the text keeps its white space.
+|===
+
+A row that cannot be converted stops the transform. When the transform has
error handling, the row goes to the error hop instead.
+
+To convert rows on several threads, give the transform more copies: each copy
converts the rows it gets on a thread of its own.
+
+NOTE: Before Hop 2.21, the transform had a *Use Parallelism* option that
converted rows on a thread pool shared by the whole JVM. That option is gone: a
transform that had it on converts its rows on one thread per copy now. Set the
number of copies to use more threads.
diff --git a/integration-tests/transforms/0127-html2text.hpl
b/integration-tests/transforms/0127-html2text.hpl
new file mode 100644
index 0000000000..a382e5fbe0
--- /dev/null
+++ b/integration-tests/transforms/0127-html2text.hpl
@@ -0,0 +1,212 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0127-html2text</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Every row reaches the output, also a row with blank or no
HTML: it gets an empty text. With one copy and with three.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/13 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/13 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>html rows</from>
+ <to>HTML to text</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>html rows</from>
+ <to>spread over the copies</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>spread over the copies</from>
+ <to>HTML to text in 3 copies</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>HTML to text</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>HTML to text in 3 copies</from>
+ <to>validate copies</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>html rows</name>
+ <type>DataGrid</type>
+ <description>HTML, HTML with only spaces, and no HTML at all
(null).</description>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item><p>Hello <b>world</b></p></item>
+ </line>
+ <line>
+ <item>2</item>
+ <item> </item>
+ </line>
+ <line>
+ <item>3</item>
+ <item/>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>html</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>HTML to text</name>
+ <type>Html2Text</type>
+ <description>One copy.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <htmlField>html</htmlField>
+ <outputField>text</outputField>
+ <safelistType>basic</safelistType>
+ <cleanOnly>N</cleanOnly>
+ <normalisedText>N</normalisedText>
+ <attributes/>
+ <GUI>
+ <xloc>336</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>spread over the copies</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>336</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>HTML to text in 3 copies</name>
+ <type>Html2Text</type>
+ <description>Three copies: the rows can come out in another
order.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>3</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <htmlField>html</htmlField>
+ <outputField>text</outputField>
+ <safelistType>basic</safelistType>
+ <cleanOnly>N</cleanOnly>
+ <normalisedText>N</normalisedText>
+ <attributes/>
+ <GUI>
+ <xloc>560</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>784</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate copies</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>784</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-0127-html2text-copies.csv
b/integration-tests/transforms/datasets/golden-0127-html2text-copies.csv
new file mode 100644
index 0000000000..df8f51d6ea
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-0127-html2text-copies.csv
@@ -0,0 +1,4 @@
+id,text
+1,Hello world
+2,
+3,
diff --git a/integration-tests/transforms/datasets/golden-0127-html2text.csv
b/integration-tests/transforms/datasets/golden-0127-html2text.csv
new file mode 100644
index 0000000000..df8f51d6ea
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-0127-html2text.csv
@@ -0,0 +1,4 @@
+id,text
+1,Hello world
+2,
+3,
diff --git a/integration-tests/transforms/main-0127-html2text.hwf
b/integration-tests/transforms/main-0127-html2text.hwf
new file mode 100644
index 0000000000..156b8da44b
--- /dev/null
+++ b/integration-tests/transforms/main-0127-html2text.hwf
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0127-html2text</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/13 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/13 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>112</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0127-html2text UNIT</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0127-html2text UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>304</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0127-html2text UNIT</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-0127-html2text-copies.json
b/integration-tests/transforms/metadata/dataset/golden-0127-html2text-copies.json
new file mode 100644
index 0000000000..7598c6fc4d
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-0127-html2text-copies.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-0127-html2text-copies.csv",
+ "name": "golden-0127-html2text-copies",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "id",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "text",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/dataset/golden-0127-html2text.json
b/integration-tests/transforms/metadata/dataset/golden-0127-html2text.json
new file mode 100644
index 0000000000..db6d97ed48
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-0127-html2text.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-0127-html2text.csv",
+ "name": "golden-0127-html2text",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "id",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "text",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git a/integration-tests/transforms/metadata/unit-test/0127-html2text
UNIT.json b/integration-tests/transforms/metadata/unit-test/0127-html2text
UNIT.json
new file mode 100644
index 0000000000..977f2ebfc9
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0127-html2text UNIT.json
@@ -0,0 +1,49 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "text",
+ "data_set_field": "text"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "data_set_name": "golden-0127-html2text",
+ "transform_name": "validate"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "text",
+ "data_set_field": "text"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "data_set_name": "golden-0127-html2text-copies",
+ "transform_name": "validate copies"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0127-html2text UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0127-html2text.hpl"
+}
diff --git
a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2Text.java
b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2Text.java
index e73b4a80bb..6ab91bd2f6 100644
---
a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2Text.java
+++
b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2Text.java
@@ -18,36 +18,27 @@
package org.apache.hop.pipeline.transforms.html2text;
import static java.lang.System.arraycopy;
-import static java.util.concurrent.ForkJoinPool.commonPool;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.trim;
import static org.apache.hop.core.util.Utils.isEmpty;
-import static org.jsoup.Jsoup.clean;
-import static org.jsoup.Jsoup.parseBodyFragment;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hop.core.exception.HopException;
-import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.jsoup.nodes.Document;
-import org.jsoup.safety.Safelist;
+/**
+ * Turns the HTML of a field into text, or into cleaned HTML. Like every
transform, a copy reads,
+ * converts and writes its rows on its own thread, in the order they come in:
to convert on more
+ * threads, run more copies.
+ */
public class Html2Text extends BaseTransform<Html2TextMeta, Html2TextData> {
public static final int OVER_ALLOCATE_SIZE = 10;
private static final Class<?> PKG = Html2Text.class; // For Translator
- private final int cores = Runtime.getRuntime().availableProcessors();
- private final int maxJobs = cores * 100;
- private final ExecutorService executor = commonPool();
- private final AtomicInteger jobs = new AtomicInteger(0);
-
- private final Object lock = new Object();
+ private static final String ERROR_CODE = "Html2Text001";
public Html2Text(
TransformMeta transformMeta,
@@ -59,25 +50,11 @@ public class Html2Text extends BaseTransform<Html2TextMeta,
Html2TextData> {
super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
}
- private void finish() throws HopException {
- // Wait until all jobs are finished.
- synchronized (lock) {
- while (jobs.get() > 0) {
- try {
- lock.wait();
- } catch (InterruptedException e) {
- throw new HopException("Waiting for jobs to finished interrupted",
e);
- }
- }
- }
- setOutputDone();
- }
-
@Override
public boolean processRow() throws HopException {
Object[] r = getRow(); // Get row from input rowset & set row busy!
if (r == null) { // no more input to be expected...
- finish();
+ setOutputDone();
return false;
}
@@ -98,98 +75,37 @@ public class Html2Text extends BaseTransform<Html2TextMeta,
Html2TextData> {
}
// cache the position of the field
cacheIndexPositions();
- } // End If first
- String errorMessage = null;
+ // The parser, and the safelist and cleaner when cleaning, are made once
+ data.converter =
+ new HtmlConverter(meta.isCleanOnly(), meta.getSafelistType(),
meta.isNormalisedText());
+ } // End If first
- process(r);
+ if (isRowLevel()) {
+ logRowlevel(
+ BaseMessages.getString(
+ PKG,
+ "Html2Text.LineNumber",
+ getLinesRead() + " : " + getInputRowMeta().getString(r)));
+ }
try {
- if (isRowLevel()) {
- logRowlevel(
- BaseMessages.getString(
- PKG,
- "Html2Text.LineNumber",
- getLinesRead() + " : " + getInputRowMeta().getString(r)));
- }
+ putRow(data.outputRowMeta, outputRow(r, convert(r)));
} catch (Exception e) {
- if (getTransformMeta().isDoingErrorHandling()) {
- errorMessage = e.toString();
- } else {
- logError(BaseMessages.getString(PKG,
"Html2Text.ErrorInTransformRunning") + e.getMessage());
- setErrors(1);
- stopAll();
- setOutputDone(); // signal end to receiver(s)
- return false;
+ if (!getTransformMeta().isDoingErrorHandling()) {
+ throw new HopException(
+ BaseMessages.getString(PKG, "Html2Text.ErrorInTransformRunning") +
e.getMessage(), e);
}
-
// Add this row to the error output of this transform
- putError(getInputRowMeta(), r, 1, errorMessage, meta.getHtmlField(),
"Html2Text001");
+ putError(getInputRowMeta(), r, 1, e.toString(), meta.getHtmlField(),
ERROR_CODE);
}
-
return true;
}
- private void process(Object[] inputRow) throws HopTransformException {
- String html = trim((String) inputRow[data.indexOfHtmlField]);
- if (isBlank(html)) {
- return;
- }
-
- if (meta.isParallelism()) {
- processAsync(html, inputRow);
- } else {
- processSync(html, inputRow);
- }
- }
-
- private void processAsync(String html, Object[] inputRow) throws
HopTransformException {
- synchronized (lock) {
- while (jobs.get() > maxJobs) {
- try {
- lock.wait();
- } catch (InterruptedException e) {
- throw new HopTransformException("Error waiting for async processing
lock release", e);
- }
- }
- }
-
- executor.submit(
- () -> {
- try {
- processSync(html, inputRow);
- jobs.decrementAndGet();
-
- synchronized (lock) {
- lock.notifyAll();
- }
- } catch (Exception e) {
- logError(e.getMessage(), e);
- }
- });
-
- jobs.incrementAndGet();
- }
-
- private void processSync(String html, Object[] inputRow) throws
HopTransformException {
-
- Document document = parseBodyFragment(html);
-
- if (meta.isCleanOnly()) {
- Safelist safelist;
- switch (meta.getSafelistType()) {
- case SIMPLE_TEXT -> safelist = Safelist.simpleText();
- case BASIC_WITH_IMAGES -> safelist = Safelist.basicWithImages();
- case NONE -> safelist = Safelist.none();
- case RELAXED -> safelist = Safelist.relaxed();
- default -> safelist = Safelist.basic();
- }
-
- processOutput(clean(document.html(), safelist), inputRow);
- } else {
- var text = meta.isNormalisedText() ? document.text() :
document.wholeText();
- processOutput(text, inputRow);
- }
+ /** The text of the row's HTML; an empty text when the HTML is empty or
null. */
+ private String convert(Object[] inputRow) throws HopException {
+ String html = trim(getInputRowMeta().getString(inputRow,
data.indexOfHtmlField));
+ return isBlank(html) ? "" : data.converter.convert(html);
}
private void cacheIndexPositions() throws HopException {
@@ -213,18 +129,10 @@ public class Html2Text extends
BaseTransform<Html2TextMeta, Html2TextData> {
}
}
- private void processOutput(String output, Object[] inputRow) throws
HopTransformException {
+ private Object[] outputRow(Object[] inputRow, String output) {
Object[] outputRow = new Object[inputRow.length + 1 + OVER_ALLOCATE_SIZE];
arraycopy(inputRow, 0, outputRow, 0, inputRow.length);
-
outputRow[data.indexOfOutputField] = output;
- putRow(data.outputRowMeta, outputRow);
- }
-
- @Override
- public void putRow(IRowMeta rowMeta, Object[] row) throws
HopTransformException {
- synchronized (this) {
- super.putRow(rowMeta, row);
- }
+ return outputRow;
}
}
diff --git
a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextData.java
b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextData.java
index 8df4c6b5e6..b84f11c80e 100644
---
a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextData.java
+++
b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextData.java
@@ -28,6 +28,9 @@ public class Html2TextData extends BaseTransformData
implements ITransformData {
public IRowMeta outputRowMeta;
public int NrPrevFields;
+ /** Converts the HTML of a row; made on the first row. */
+ HtmlConverter converter;
+
public Html2TextData() {
super();
indexOfHtmlField = -1;
diff --git
a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextDialog.java
b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextDialog.java
index 9d3612f90c..889dbb0c73 100644
---
a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextDialog.java
+++
b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextDialog.java
@@ -72,7 +72,6 @@ public class Html2TextDialog extends BaseTransformDialog
implements ITransformDi
private Button wCleanOnly;
private Button wNormalisedText;
- private Button wParallelism;
public Html2TextDialog(
Shell parent, IVariables variables, Object in, PipelineMeta
pipelineMeta, String sname) {
@@ -250,33 +249,6 @@ public class Html2TextDialog extends BaseTransformDialog
implements ITransformDi
}
});
- // Parallelism
- Label wlParallelism = new Label(wContent, RIGHT);
- wlParallelism.setText(getString(PKG, "Html2TextDialog.Parallelism.Label"));
- PropsUi.setLook(wlParallelism);
- FormData fdlParallelism = new FormData();
- fdlParallelism.left = new FormAttachment(0, 0);
- fdlParallelism.top = new FormAttachment(wSafelistType, margin);
- fdlParallelism.right = new FormAttachment(middle, -margin);
- wlParallelism.setLayoutData(fdlParallelism);
-
- wParallelism = new Button(wContent, CHECK);
- wParallelism.setSelection(input.isParallelism());
- PropsUi.setLook(wParallelism);
- wParallelism.setToolTipText(getString(PKG,
"Html2TextDialog.Parallelism.Tooltip"));
- FormData fdParallelism = new FormData();
- fdParallelism.left = new FormAttachment(middle, -margin);
- fdParallelism.top = new FormAttachment(wlParallelism, 0, SWT.CENTER);
- fdParallelism.right = new FormAttachment(100, 0);
- wParallelism.setLayoutData(fdParallelism);
- wParallelism.addSelectionListener(
- new SelectionAdapter() {
- @Override
- public void widgetSelected(SelectionEvent e) {
- input.setChanged();
- }
- });
-
wContent.pack();
Rectangle bounds = wContent.getBounds();
sc.setContent(wContent);
@@ -296,7 +268,6 @@ public class Html2TextDialog extends BaseTransformDialog
implements ITransformDi
/** Copy information from the meta-data input to the dialog fields. */
public void getData() {
wHtmlFieldName.setText(Const.NVL(input.getHtmlField(), ""));
- wParallelism.setSelection(input.isParallelism());
wCleanOnly.setSelection(input.isCleanOnly());
wNormalisedText.setSelection(input.isNormalisedText());
wSafelistType.setText(input.getSafelistType().getDescription());
@@ -319,7 +290,6 @@ public class Html2TextDialog extends BaseTransformDialog
implements ITransformDi
input.setOutputField(wOutputField.getText());
input.setCleanOnly(wCleanOnly.getSelection());
input.setNormalisedText(wNormalisedText.getSelection());
- input.setParallelism(wParallelism.getSelection());
transformName = wTransformName.getText(); // return value
diff --git
a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextMeta.java
b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextMeta.java
index 19a135bf30..875ba52b61 100644
---
a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextMeta.java
+++
b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/Html2TextMeta.java
@@ -65,9 +65,6 @@ public class Html2TextMeta extends
BaseTransformMeta<Html2Text, Html2TextData> {
@HopMetadataProperty(key = "cleanOnly")
private boolean cleanOnly;
- @HopMetadataProperty(key = "parallelism")
- private boolean parallelism;
-
@HopMetadataProperty(key = "normalisedText")
private boolean normalisedText;
diff --git
a/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/HtmlConverter.java
b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/HtmlConverter.java
new file mode 100644
index 0000000000..7ea2aeaf01
--- /dev/null
+++
b/plugins/transforms/html2text/src/main/java/org/apache/hop/pipeline/transforms/html2text/HtmlConverter.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.html2text;
+
+import org.apache.hop.pipeline.transforms.html2text.Html2TextMeta.SafelistType;
+import org.jsoup.nodes.Document;
+import org.jsoup.parser.Parser;
+import org.jsoup.safety.Cleaner;
+import org.jsoup.safety.Safelist;
+
+/**
+ * Turns HTML into text, or into cleaned HTML, the way the HTML to text
transform is set up.
+ *
+ * <p>The static {@code Jsoup.parseBodyFragment()} and {@code Jsoup.clean()}
build a new parser, a
+ * new safelist and a new cleaner for every call. Here they are made once and
used for every row.
+ * The outcome is the same.
+ *
+ * <p>One converter per transform copy, used by the thread of that copy only.
+ */
+final class HtmlConverter {
+
+ private final Parser parser = Parser.htmlParser();
+
+ /** Null when the HTML is turned into text. */
+ private final Cleaner cleaner;
+
+ private final boolean normalisedText;
+
+ /**
+ * @param cleanOnly true to keep the HTML and remove what the safelist does
not allow
+ * @param safelistType the safelist for cleaning; the basic one when null
+ * @param normalisedText for text: true for normalised whitespace, false for
the text as it is
+ */
+ HtmlConverter(boolean cleanOnly, SafelistType safelistType, boolean
normalisedText) {
+ this.cleaner = cleanOnly ? new Cleaner(safelist(safelistType)) : null;
+ this.normalisedText = normalisedText;
+ }
+
+ private static Safelist safelist(SafelistType type) {
+ if (type == null) {
+ return Safelist.basic();
+ }
+ return switch (type) {
+ case SIMPLE_TEXT -> Safelist.simpleText();
+ case BASIC_WITH_IMAGES -> Safelist.basicWithImages();
+ case NONE -> Safelist.none();
+ case RELAXED -> Safelist.relaxed();
+ default -> Safelist.basic();
+ };
+ }
+
+ /**
+ * @param html the HTML, not null
+ * @return the text of the HTML, or the cleaned HTML
+ */
+ String convert(String html) {
+ Document document = parseBodyFragment(html);
+ if (cleaner != null) {
+ // As Jsoup.clean(document.html(), safelist): the document as HTML,
parsed again and cleaned
+ return cleaner.clean(parseBodyFragment(document.html())).body().html();
+ }
+ return normalisedText ? document.text() : document.wholeText();
+ }
+
+ /** What Jsoup.parseBodyFragment(html) does, with the parser of this
converter. */
+ private Document parseBodyFragment(String html) {
+ Document document = Document.createShell("");
+ document.body().appendChildren(parser.parseFragmentInput(html,
document.body(), ""));
+ return document;
+ }
+}
diff --git
a/plugins/transforms/html2text/src/main/resources/org/apache/hop/pipeline/transforms/html2text/messages/messages_en_US.properties
b/plugins/transforms/html2text/src/main/resources/org/apache/hop/pipeline/transforms/html2text/messages/messages_en_US.properties
index 8a2c0387bc..58cb376655 100644
---
a/plugins/transforms/html2text/src/main/resources/org/apache/hop/pipeline/transforms/html2text/messages/messages_en_US.properties
+++
b/plugins/transforms/html2text/src/main/resources/org/apache/hop/pipeline/transforms/html2text/messages/messages_en_US.properties
@@ -52,9 +52,7 @@ Html2TextDialog.SafelistType.basicWithImages=Basic with images
Html2TextDialog.OutputField.Label=Output text field
Html2TextDialog.CleanOnly.Label=Clean HTML Only
Html2TextDialog.NormalisedText.Label=Normalize Text
-Html2TextDialog.Parallelism.Label=Use Parallelism
Html2Text.Error.OutputFieldMissing=Output field is missing!
Html2TextDialog.NormalisedText.Tooltip=Normalize the white space in the
resulting text
-Html2TextDialog.Parallelism.Tooltip=Process the rows in parallel
Html2TextMeta.CheckResult.OutputFieldMissing=Output field is missing!
Html2TextMeta.CheckResult.OutputFieldOK=Output field was specified.
diff --git
a/plugins/transforms/html2text/src/main/resources/org/apache/hop/pipeline/transforms/html2text/messages/messages_pt_BR.properties
b/plugins/transforms/html2text/src/main/resources/org/apache/hop/pipeline/transforms/html2text/messages/messages_pt_BR.properties
index 7e4e47b9f8..8ccc2675a1 100644
---
a/plugins/transforms/html2text/src/main/resources/org/apache/hop/pipeline/transforms/html2text/messages/messages_pt_BR.properties
+++
b/plugins/transforms/html2text/src/main/resources/org/apache/hop/pipeline/transforms/html2text/messages/messages_pt_BR.properties
@@ -26,7 +26,6 @@ Html2Text.Log.ErrorFindingField=Erro procurando campo:
Html2Text.Log.UnexpectedError=Erro inesperado em ''
Html2TextDialog.FailedToGetFields.DialogMessage=Não foi possível obter campos
de transformações anteriores devido a um erro
Html2TextDialog.FailedToGetFields.DialogTitle=Falha na obtenção dos campos
-Html2TextDialog.Parallelism.Label=Usar paralelismo
Html2TextDialog.SafelistType.basic=Básico
Html2TextDialog.SafelistType.none=nenhum
Html2TextDialog.Shell.Title=HTML para texto
diff --git
a/plugins/transforms/html2text/src/main/samples/transforms/html2text.hpl
b/plugins/transforms/html2text/src/main/samples/transforms/html2text.hpl
index fe09064f08..916829c8bb 100644
--- a/plugins/transforms/html2text/src/main/samples/transforms/html2text.hpl
+++ b/plugins/transforms/html2text/src/main/samples/transforms/html2text.hpl
@@ -136,7 +136,6 @@ HTML documents before text extraction.</note>
<outputField>clean_html</outputField>
<safelistType>basic</safelistType>
<cleanOnly>Y</cleanOnly>
- <parallelism>N</parallelism>
<attributes/>
<GUI>
<xloc>864</xloc>
@@ -158,7 +157,6 @@ HTML documents before text extraction.</note>
<outputField>text</outputField>
<safelistType>basic</safelistType>
<cleanOnly>N</cleanOnly>
- <parallelism>N</parallelism>
<attributes/>
<GUI>
<xloc>1024</xloc>
@@ -180,7 +178,6 @@ HTML documents before text extraction.</note>
<outputField>text</outputField>
<safelistType>basic</safelistType>
<cleanOnly>N</cleanOnly>
- <parallelism>N</parallelism>
<attributes/>
<GUI>
<xloc>1024</xloc>
diff --git
a/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/Html2TextMetaTest.java
b/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/Html2TextMetaTest.java
index 041e401389..5d597ae29e 100644
---
a/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/Html2TextMetaTest.java
+++
b/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/Html2TextMetaTest.java
@@ -45,6 +45,6 @@ class Html2TextMetaTest {
Assertions.assertEquals(Html2TextMeta.SafelistType.BASIC,
meta.getSafelistType());
Assertions.assertTrue(meta.isCleanOnly());
Assertions.assertTrue(meta.isNormalisedText());
- Assertions.assertTrue(meta.isParallelism());
+ // transform.xml still has the parallelism option that is gone: a
transform saved with it loads
}
}
diff --git
a/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/Html2TextTest.java
b/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/Html2TextTest.java
new file mode 100644
index 0000000000..4848736cb9
--- /dev/null
+++
b/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/Html2TextTest.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.html2text;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+class Html2TextTest {
+
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ private TransformMockHelper<Html2TextMeta, Html2TextData> mockHelper;
+
+ @BeforeEach
+ void setUp() {
+ mockHelper =
+ new TransformMockHelper<>("HTML to text", Html2TextMeta.class,
Html2TextData.class);
+ when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(mockHelper.iLogChannel);
+ when(mockHelper.pipeline.isRunning()).thenReturn(true);
+ when(mockHelper.transformMeta.getCopies(any())).thenReturn(1);
+ when(mockHelper.transformMeta.isPartitioned()).thenReturn(false);
+ when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(false);
+ }
+
+ @AfterEach
+ void tearDown() {
+ mockHelper.cleanUp();
+ }
+
+ @Test
+ void theTextOfTheHtmlGoesIntoTheOutputField() throws Exception {
+ Html2Text transform = createTransform(meta(), false);
+ assertTrue(transform.init());
+
+ BlockingRowSet output =
+ run(transform, row(1L, "<p>Hello <b>world</b> & all</p>"), row(2L,
"no tags"));
+
+ Object[] first = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(1L, first[0]);
+ assertEquals("<p>Hello <b>world</b> & all</p>", first[1]);
+ assertEquals("Hello world & all", first[2]);
+ assertEquals("no tags", output.getRowWait(1, TimeUnit.SECONDS)[2]);
+ assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS));
+ transform.dispose();
+ }
+
+ /** A row with a null or blank HTML field used to disappear from the stream.
*/
+ @Test
+ void aRowWithEmptyHtmlPassesOnWithAnEmptyText() throws Exception {
+ Html2Text transform = createTransform(meta(), false);
+ assertTrue(transform.init());
+
+ BlockingRowSet output =
+ run(transform, row(1L, null), row(2L, " "), row(3L, "<p>text</p>"),
row(4L, ""));
+
+ for (long id : new long[] {1L, 2L}) {
+ Object[] row = output.getRowWait(1, TimeUnit.SECONDS);
+ assertNotNull(row, "row " + id + " was dropped");
+ assertEquals(id, row[0]);
+ assertEquals("", row[2]);
+ }
+ assertEquals("text", output.getRowWait(1, TimeUnit.SECONDS)[2]);
+ Object[] last = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(4L, last[0]);
+ assertEquals("", last[2]);
+ assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS));
+ transform.dispose();
+ }
+
+ @Test
+ void aFailingRowStopsTheTransform() {
+ Html2Text transform = createTransform(meta(), true);
+ assertTrue(transform.init());
+
+ HopException e = assertThrows(HopException.class, () -> run(transform,
row(1L, "<p>x</p>")));
+ assertTrue(e.getMessage().contains("no room for this row"));
+ transform.dispose();
+ }
+
+ /**
+ * Like every transform, a copy writes each row on its own thread, before
processRow() returns, in
+ * the order the rows came in. To convert on more threads, run more copies.
+ */
+ @Test
+ void eachRowIsWrittenOnTheTransformThreadBeforeProcessRowReturns() throws
Exception {
+ List<Thread> writers = new ArrayList<>();
+ Html2TextMeta meta = meta();
+ when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+ Html2Text transform =
+ new Html2Text(
+ mockHelper.transformMeta,
+ meta,
+ new Html2TextData(),
+ 0,
+ mockHelper.pipelineMeta,
+ mockHelper.pipeline) {
+ @Override
+ public void putRow(IRowMeta rowMeta, Object[] row) throws
HopTransformException {
+ writers.add(Thread.currentThread());
+ super.putRow(rowMeta, row);
+ }
+ };
+ assertTrue(transform.init());
+ Object[][] rows = new Object[100][];
+ for (int i = 0; i < rows.length; i++) {
+ rows[i] = row((long) i, i % 10 == 0 ? null : "<p>row <i>" + i +
"</i></p>");
+ }
+ BlockingRowSet output = connect(transform, rows);
+
+ for (int i = 0; i < rows.length; i++) {
+ assertTrue(transform.processRow());
+ assertEquals(i + 1, writers.size());
+ assertSame(Thread.currentThread(), writers.get(i));
+ Object[] written = output.getRowImmediate();
+ assertNotNull(written, "row " + i + " was not written yet");
+ assertEquals((long) i, written[0]);
+ }
+ assertFalse(transform.processRow());
+ transform.dispose();
+ }
+
+ /** Several copies at once: every copy converts its own rows. */
+ @Test
+ void copiesRunningAtTheSameTimeDoNotMixTheirRows() throws Exception {
+ int copies = 6;
+ int rowsPerCopy = 300;
+ Html2TextMeta meta = meta();
+ when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+ ExecutorService pool = Executors.newFixedThreadPool(copies);
+ try {
+ CountDownLatch start = new CountDownLatch(1);
+ List<Future<BlockingRowSet>> outputs = new ArrayList<>();
+ for (int copy = 0; copy < copies; copy++) {
+ int copyNr = copy;
+ outputs.add(
+ pool.submit(
+ () -> {
+ Object[][] rows = new Object[rowsPerCopy][];
+ for (int r = 0; r < rowsPerCopy; r++) {
+ rows[r] = row(copyNr * 10_000L + r, "<b>copy " + copyNr +
"</b> row " + r);
+ }
+ Html2Text transform = newTransform(meta, false);
+ assertTrue(transform.init());
+ start.await();
+ return run(transform, rows);
+ }));
+ }
+ start.countDown();
+ for (int copy = 0; copy < copies; copy++) {
+ BlockingRowSet output = outputs.get(copy).get(2, TimeUnit.MINUTES);
+ for (int r = 0; r < rowsPerCopy; r++) {
+ Object[] row = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(copy * 10_000L + r, row[0]);
+ assertEquals("copy " + copy + " row " + r, row[2]);
+ }
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+
+ private static Html2TextMeta meta() {
+ Html2TextMeta meta = new Html2TextMeta();
+ meta.setHtmlField("html");
+ meta.setOutputField("text");
+ meta.setCleanOnly(false);
+ meta.setNormalisedText(true);
+ return meta;
+ }
+
+ private static Object[] row(Long id, String html) {
+ return new Object[] {id, html};
+ }
+
+ /** Gives the transform an input with the rows and an empty output, and
returns the output. */
+ private static BlockingRowSet connect(Html2Text transform, Object[]... rows)
{
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("id"));
+ rowMeta.addValueMeta(new ValueMetaString("html"));
+
+ BlockingRowSet input = new BlockingRowSet(rows.length + 10);
+ input.setThreadNameFromToCopy("previous", 0, "HTML to text", 0);
+ input.setRowMeta(rowMeta);
+ for (Object[] row : rows) {
+ input.putRow(rowMeta, row);
+ }
+ input.setDone();
+
+ BlockingRowSet output = new BlockingRowSet(rows.length + 10);
+ output.setThreadNameFromToCopy("HTML to text", 0, "next", 0);
+
+ transform.addRowSetToInputRowSets(input);
+ transform.addRowSetToOutputRowSets(output);
+ return output;
+ }
+
+ /** Feeds the rows through the transform until it reports the end of its
input. */
+ private static BlockingRowSet run(Html2Text transform, Object[]... rows)
throws HopException {
+ BlockingRowSet output = connect(transform, rows);
+ int calls = 0;
+ boolean more = true;
+ while (more) {
+ more = transform.processRow();
+ calls++;
+ }
+ assertEquals(rows.length + 1, calls);
+ assertFalse(more);
+ return output;
+ }
+
+ private Html2Text createTransform(Html2TextMeta meta, boolean failOnWrite) {
+ when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+ return newTransform(meta, failOnWrite);
+ }
+
+ /** Without stubbing: safe to call from several threads. */
+ private Html2Text newTransform(Html2TextMeta meta, boolean failOnWrite) {
+ Html2TextData data = new Html2TextData();
+ if (!failOnWrite) {
+ return new Html2Text(
+ mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta,
mockHelper.pipeline);
+ }
+ return new Html2Text(
+ mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta,
mockHelper.pipeline) {
+ @Override
+ public void putRow(IRowMeta rowMeta, Object[] row) throws
HopTransformException {
+ throw new HopTransformException("no room for this row");
+ }
+ };
+ }
+}
diff --git
a/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/HtmlConverterTest.java
b/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/HtmlConverterTest.java
new file mode 100644
index 0000000000..0e1fd0f6ce
--- /dev/null
+++
b/plugins/transforms/html2text/src/test/java/org/apache/hop/pipeline/transforms/html2text/HtmlConverterTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.html2text;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.List;
+import org.apache.hop.pipeline.transforms.html2text.Html2TextMeta.SafelistType;
+import org.jsoup.Jsoup;
+import org.jsoup.safety.Safelist;
+import org.junit.jupiter.api.Test;
+
+/**
+ * {@link HtmlConverter} keeps one parser and one cleaner. Its outcome has to
be that of the static
+ * jsoup calls the transform used to make for every row.
+ */
+class HtmlConverterTest {
+
+ private static final List<String> HTML =
+ List.of(
+
"<html><head><title>Title</title></head><body><h1>Head</h1><p><b>bold</b> &
"
+ + "<i>italic</i></p><ul><li>item one</li><li>item
two</li></ul></body></html>",
+ "plain text, no tags",
+ "<p>broken <b>markup<p>second <i>paragraph",
+ "<p>a script <script>alert(1)</script> and a <a
href='http://example.com' "
+ + "onclick='x()'>link</a></p>",
+ "entities < > é € and accents é ü ñ",
+ "<table><tr><td>cell<td>two</table><div><p>nested <span
style='color:red'>span</span>"
+ + "</div><img src='a.png' alt='picture'><br>line",
+ "<pre> keep these\n spaces </pre><p> but not\n these </p>",
+ "<!-- a comment --><style>p { color: red }</style><p>after</p>",
+ " ",
+ "<p></p>");
+
+ /** What the transform did before, for text. */
+ private static String textBefore(String html, boolean normalised) {
+ return normalised
+ ? Jsoup.parseBodyFragment(html).text()
+ : Jsoup.parseBodyFragment(html).wholeText();
+ }
+
+ /** What the transform did before, for cleaning. */
+ private static String cleanBefore(String html, Safelist safelist) {
+ return Jsoup.clean(Jsoup.parseBodyFragment(html).html(), safelist);
+ }
+
+ @Test
+ void wholeTextIsTheSameAsBefore() {
+ HtmlConverter converter = new HtmlConverter(false, SafelistType.BASIC,
false);
+ for (String html : HTML) {
+ assertEquals(textBefore(html, false), converter.convert(html), html);
+ }
+ }
+
+ @Test
+ void normalisedTextIsTheSameAsBefore() {
+ HtmlConverter converter = new HtmlConverter(false, SafelistType.BASIC,
true);
+ for (String html : HTML) {
+ assertEquals(textBefore(html, true), converter.convert(html), html);
+ }
+ }
+
+ @Test
+ void cleaningIsTheSameAsBeforeForEverySafelist() {
+ Object[][] safelists = {
+ {SafelistType.NONE, Safelist.none()},
+ {SafelistType.SIMPLE_TEXT, Safelist.simpleText()},
+ {SafelistType.BASIC, Safelist.basic()},
+ {SafelistType.BASIC_WITH_IMAGES, Safelist.basicWithImages()},
+ {SafelistType.RELAXED, Safelist.relaxed()},
+ // no safelist chosen: the basic one
+ {null, Safelist.basic()}
+ };
+ for (Object[] safelist : safelists) {
+ HtmlConverter converter = new HtmlConverter(true, (SafelistType)
safelist[0], false);
+ for (String html : HTML) {
+ assertEquals(
+ cleanBefore(html, (Safelist) safelist[1]),
+ converter.convert(html),
+ safelist[0] + ": " + html);
+ }
+ }
+ }
+
+ /** One converter, so one parser, for row after row: nothing of a row stays
for the next one. */
+ @Test
+ void oneConverterGivesEveryRowItsOwnText() {
+ HtmlConverter text = new HtmlConverter(false, SafelistType.BASIC, true);
+ HtmlConverter clean = new HtmlConverter(true, SafelistType.BASIC, false);
+ for (int i = 0; i < 4000; i++) {
+ String html = "<p>row <b>" + i + "</b></p>" + HTML.get(i % HTML.size());
+ assertEquals(textBefore(html, true), text.convert(html), html);
+ assertEquals(cleanBefore(html, Safelist.basic()), clean.convert(html),
html);
+ }
+ }
+}