This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 0769b95426 Text chunker: AsciiDoc blocks, Markdown code fences,
skipped heading … (#9363)
0769b95426 is described below
commit 0769b95426cd85e20e9a5e3fe3737f44e5f611d7
Author: Bart Maertens <[email protected]>
AuthorDate: Thu Nov 12 10:30:39 2026 +0100
Text chunker: AsciiDoc blocks, Markdown code fences, skipped heading …
(#9363)
* Text chunker: AsciiDoc blocks, Markdown code fences, skipped heading
levels and content detection break Structure sections, fixes #9282
* Text chunker: two-line AsciiDoc titles, verbatim blocks and fences in
detection, single scan per syntax, #9282
---
.../pages/pipeline/transforms/textchunker.adoc | 6 +-
.../0115-text-chunker-structure-headings.hpl | 178 ++++++++++++++++++++
.../golden-text-chunker-structure-headings.csv | 75 +++++++++
.../transforms/main-0115-text-chunker.hwf | 3 +
.../golden-text-chunker-structure-headings.json | 47 ++++++
.../0115-text-chunker-structure-headings UNIT.json | 48 ++++++
.../chunker/document/AsciiDocDocumentParser.java | 186 ++++++++++++++++++++-
.../chunker/document/DocumentParserRegistry.java | 108 +++++++++---
.../chunker/document/HeadingParserSupport.java | 89 +++++-----
.../chunker/document/MarkdownDocumentParser.java | 77 ++++++++-
.../chunker/document/hopxml/HopXmlSupport.java | 97 ++++++++++-
.../document/DocumentParserRegistryTest.java | 100 +++++++++++
.../chunker/document/HeadingParsersTest.java | 167 ++++++++++++++++++
13 files changed, 1103 insertions(+), 78 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc
index e329402eca..f01ed986ae 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc
@@ -53,10 +53,10 @@ The Structure strategy selects a parser from the content
type:
|===
|Content type |Parsed structure
-|Auto|Infer from the content type field, or from the text itself.
+|Auto|Infer from the content type field, or from the text itself: a pipeline
or workflow when the XML root element is `pipeline` or `workflow`, metadata
when the text is a JSON object, AsciiDoc when it starts with a `= Title`
document title, otherwise Markdown or AsciiDoc by which has more headings, and
on a tie by the first heading.
|Plain|Blank-line separated blocks.
-|Markdown|`#` heading levels.
-|AsciiDoc|`=` heading levels.
+|Markdown|`#` heading levels. Lines in a fenced code block are not headings. A
skipped level nests under the nearest heading above it, and the closing `#`
sequence is not part of the title.
+|AsciiDoc|`=` heading levels and two-line titles (a title underlined with `=`,
`-`, `~`, `^` or `+`). Lines in a delimited block (`====`, `----`, `////`, ...)
are not headings.
|Pipeline|A Hop pipeline (`.hpl`): one section per transform, plus the hops
between them.
|Workflow|A Hop workflow (`.hwf`): one section per action, plus the hops
between them.
|Metadata|A Hop metadata JSON document: an overview plus one section per entry.
diff --git
a/integration-tests/transforms/0115-text-chunker-structure-headings.hpl
b/integration-tests/transforms/0115-text-chunker-structure-headings.hpl
new file mode 100644
index 0000000000..22aa2dc438
--- /dev/null
+++ b/integration-tests/transforms/0115-text-chunker-structure-headings.hpl
@@ -0,0 +1,178 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0115-text-chunker-structure-headings</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Structure chunking: AsciiDoc delimited blocks and Markdown
code fences are not headings, skipped heading levels keep their parent, closing
hashes are dropped and Auto picks the right parser (#9282).</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/07 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/07 12:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop> <from>Document</from> <to>Chunk</to> <enabled>Y</enabled> </hop>
+ <hop> <from>Chunk</from> <to>OUTPUT</to> <enabled>Y</enabled> </hop>
+ </order>
+ <transform>
+ <name>Document</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>asciidoc-example-block</item>
+ <item>ASCIIDOC</item>
+ <item>= Title Intro. == Section Before the
block. ==== Example text inside the block. ==== After
the block. == Next More text.</item>
+ </line>
+ <line>
+ <item>asciidoc-example-block-auto</item>
+ <item>AUTO</item>
+ <item>= Title Intro. == Section Before the
block. ==== Example text inside the block. ==== After
the block. == Next More text.</item>
+ </line>
+ <line>
+ <item>markdown-code-fence</item>
+ <item>MARKDOWN</item>
+ <item># Guide Intro. ```bash # download the
zip curl -O https://example.com/hop.zip ``` After the
block.</item>
+ </line>
+ <line>
+ <item>markdown-skipped-level</item>
+ <item>MARKDOWN</item>
+ <item># Guide Intro. ### Detail Detail
text.</item>
+ </line>
+ <line>
+ <item>markdown-closing-hashes</item>
+ <item>MARKDOWN</item>
+ <item># Guide # Intro. ## Install
## Install text.</item>
+ </line>
+ <line>
+ <item>auto-markdown-mentions-hop-xml</item>
+ <item>AUTO</item>
+ <item># Pipelines A <pipeline> runs transforms, a
<workflow> runs actions. ## Run Use hop-run.</item>
+ </line>
+ <line>
+ <item>auto-markdown-starts-with-brace</item>
+ <item>AUTO</item>
+ <item>{{< note >}} Read this first. {{< /note
>}} # Guide Intro. ## Next More
text.</item>
+ </line>
+ <line>
+ <item>auto-markdown-equals-line</item>
+ <item>AUTO</item>
+ <item># Guide The answer: = 42 ##
Next More text.</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>case_id</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>content_type</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>content</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Chunk</name>
+ <type>TextChunker</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <inputField>content</inputField>
+ <outputChunkField>chunk_text</outputChunkField>
+ <chunkingStrategy>STRUCTURE</chunkingStrategy>
+ <contentType>AUTO</contentType>
+ <contentTypeField>content_type</contentTypeField>
+ <chunkSize>1000</chunkSize>
+ <chunkOverlap>0</chunkOverlap>
+ <includeMetadata>Y</includeMetadata>
+ <chunkIndexField>chunk_index</chunkIndexField>
+ <chunkStartPosField>chunk_start_position</chunkStartPosField>
+ <documentIdField>chunk_doc_id</documentIdField>
+ <chunkCountField>total_chunks</chunkCountField>
+ <attributes/>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>OUTPUT</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>544</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-text-chunker-structure-headings.csv
b/integration-tests/transforms/datasets/golden-text-chunker-structure-headings.csv
new file mode 100644
index 0000000000..ba9c7eff0f
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-text-chunker-structure-headings.csv
@@ -0,0 +1,75 @@
+case_id,chunk_text,chunk_index,chunk_start_position,total_chunks
+"asciidoc-example-block","Title
+
+Intro.",0,0,3
+"asciidoc-example-block","Title > Section
+
+Before the block.
+
+====
+Example text inside the block.
+====
+
+After the block.",1,17,3
+"asciidoc-example-block","Title > Next
+
+More text.",2,108,3
+"asciidoc-example-block-auto","Title
+
+Intro.",0,0,3
+"asciidoc-example-block-auto","Title > Section
+
+Before the block.
+
+====
+Example text inside the block.
+====
+
+After the block.",1,17,3
+"asciidoc-example-block-auto","Title > Next
+
+More text.",2,108,3
+"markdown-code-fence","Guide
+
+Intro.
+
+```bash
+# download the zip
+curl -O https://example.com/hop.zip
+```
+
+After the block.",0,0,1
+"markdown-skipped-level","Guide
+
+Intro.",0,0,2
+"markdown-skipped-level","Guide > Detail
+
+Detail text.",1,17,2
+"markdown-closing-hashes","Guide
+
+Intro.",0,0,2
+"markdown-closing-hashes","Guide > Install
+
+Install text.",1,19,2
+"auto-markdown-mentions-hop-xml","Pipelines
+
+A <pipeline> runs transforms, a <workflow> runs actions.",0,0,2
+"auto-markdown-mentions-hop-xml","Pipelines > Run
+
+Use hop-run.",1,71,2
+"auto-markdown-starts-with-brace","{{< note >}}
+Read this first.
+{{< /note >}}",0,0,3
+"auto-markdown-starts-with-brace","Guide
+
+Intro.",1,45,3
+"auto-markdown-starts-with-brace","Guide > Next
+
+More text.",2,62,3
+"auto-markdown-equals-line","Guide
+
+The answer:
+= 42",0,0,2
+"auto-markdown-equals-line","Guide > Next
+
+More text.",1,27,2
diff --git a/integration-tests/transforms/main-0115-text-chunker.hwf
b/integration-tests/transforms/main-0115-text-chunker.hwf
index e1c26aaac9..f9e7690f68 100644
--- a/integration-tests/transforms/main-0115-text-chunker.hwf
+++ b/integration-tests/transforms/main-0115-text-chunker.hwf
@@ -72,6 +72,9 @@ limitations under the License.
<test_name>
<name>0115-text-chunker-structure-split UNIT</name>
</test_name>
+ <test_name>
+ <name>0115-text-chunker-structure-headings UNIT</name>
+ </test_name>
<test_name>
<name>0117-text-chunker-redaction UNIT</name>
</test_name>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-text-chunker-structure-headings.json
b/integration-tests/transforms/metadata/dataset/golden-text-chunker-structure-headings.json
new file mode 100644
index 0000000000..4c109df54b
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-text-chunker-structure-headings.json
@@ -0,0 +1,47 @@
+{
+ "base_filename": "golden-text-chunker-structure-headings.csv",
+ "name": "golden-text-chunker-structure-headings",
+ "description": "Structure chunking of Markdown and AsciiDoc: delimited
blocks and code fences are not headings, skipped levels keep their parent,
closing hashes are dropped, Auto picks the right parser (#9282)",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "case_id",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "chunk_text",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "chunk_index",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "chunk_start_position",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "total_chunks",
+ "field_format": "0"
+ }
+ ]
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0115-text-chunker-structure-headings
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0115-text-chunker-structure-headings
UNIT.json
new file mode 100644
index 0000000000..84229d2403
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0115-text-chunker-structure-headings
UNIT.json
@@ -0,0 +1,48 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": false,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "case_id",
+ "data_set_field": "case_id"
+ },
+ {
+ "transform_field": "chunk_text",
+ "data_set_field": "chunk_text"
+ },
+ {
+ "transform_field": "chunk_index",
+ "data_set_field": "chunk_index"
+ },
+ {
+ "transform_field": "chunk_start_position",
+ "data_set_field": "chunk_start_position"
+ },
+ {
+ "transform_field": "total_chunks",
+ "data_set_field": "total_chunks"
+ }
+ ],
+ "field_order": [
+ "case_id",
+ "chunk_text",
+ "chunk_index",
+ "chunk_start_position",
+ "total_chunks"
+ ],
+ "transform_name": "OUTPUT",
+ "data_set_name": "golden-text-chunker-structure-headings"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0115-text-chunker-structure-headings UNIT",
+ "description": "Structure chunking of Markdown and AsciiDoc: delimited
blocks and code fences are not headings, skipped levels keep their parent,
closing hashes are dropped, Auto picks the right parser (#9282)",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0115-text-chunker-structure-headings.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/AsciiDocDocumentParser.java
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/AsciiDocDocumentParser.java
index 72c097d832..9003a71f6e 100644
---
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/AsciiDocDocumentParser.java
+++
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/AsciiDocDocumentParser.java
@@ -16,12 +16,37 @@
*/
package org.apache.hop.pipeline.transforms.chunker.document;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import
org.apache.hop.pipeline.transforms.chunker.document.HeadingParserSupport.Heading;
+import
org.apache.hop.pipeline.transforms.chunker.document.HeadingParserSupport.Line;
-/** Splits AsciiDoc on {@code =} headings and builds a section tree. */
+/**
+ * Splits AsciiDoc on {@code =} section titles and two-line (setext) titles,
and builds a section
+ * tree. Lines inside a delimited block ({@code ====}, {@code ----}, {@code
////}, ...) are never
+ * section titles, and a delimiter line is never a title either.
+ */
public final class AsciiDocDocumentParser implements DocumentParser {
- private static final Pattern HEADING = Pattern.compile("(?m)^(=+)\\s+(.+)$");
+ /** A section title: 1-6 {@code =}, then a space or tab and the title, on a
single line. */
+ private static final Pattern HEADING = Pattern.compile("^(={1,6})[
\\t]+(\\S.*)$");
+
+ /**
+ * A delimited block line: 4 or more of the same delimiter character, {@code
--} (open block), a
+ * {@code ```} fence, or a table delimiter such as {@code |===}.
+ */
+ private static final Pattern DELIMITER =
+ Pattern.compile("^(?:([/\\-.+=*_])\\1{3,}|--|```.*|[|,:!]={3,})$");
+
+ /** An attribute entry such as {@code :description: ...} or {@code :!toc:}.
*/
+ private static final Pattern ATTRIBUTE_ENTRY =
Pattern.compile("^:!?[\\w-]+!?:.*$");
+
+ /** The underline characters of a two-line title, from level 1 ({@code =})
to level 5. */
+ private static final String SETEXT_UNDERLINES = "=-~^+";
@Override
public ContentType getContentType() {
@@ -30,6 +55,161 @@ public final class AsciiDocDocumentParser implements
DocumentParser {
@Override
public DocumentNode parse(String text) {
- return HeadingParserSupport.parse(text, HEADING, String::length);
+ if (text == null || text.isEmpty()) {
+ return DocumentNode.root("");
+ }
+ return HeadingParserSupport.parse(text, scan(text).headings());
+ }
+
+ /**
+ * What content detection needs from one pass over an AsciiDoc text.
+ *
+ * @param headings all section titles, in order
+ * @param oneLineTitles the {@code =} titles only; Markdown has two-line
titles too, so those do
+ * not tell the syntaxes apart
+ * @param verbatimBlocks the {@code [start, end)} offsets of the closed
listing, literal,
+ * passthrough, comment and fenced blocks, where a {@code #} line is not
a Markdown heading
+ * @param startsWithDocumentTitle the first line with content, after
comments, attribute entries
+ * and block attributes, is a document title ({@code = Title})
+ */
+ record Scan(
+ List<Heading> headings,
+ List<Heading> oneLineTitles,
+ List<int[]> verbatimBlocks,
+ boolean startsWithDocumentTitle) {}
+
+ static Scan scan(String text) {
+ List<Heading> headings = new ArrayList<>();
+ List<Heading> oneLineTitles = new ArrayList<>();
+ List<int[]> verbatimBlocks = new ArrayList<>();
+ Deque<String> openBlocks = new ArrayDeque<>();
+ int verbatimStart = -1;
+ // The document title check looks at the first line with content only.
+ Boolean documentTitle = null;
+ // A text line that starts a block, and so may be the first line of a
two-line title.
+ Line titleCandidate = null;
+ boolean blockStart = true;
+
+ for (Line line : HeadingParserSupport.lines(text)) {
+ String content = line.text().stripTrailing();
+ Line previous = titleCandidate;
+ titleCandidate = null;
+ boolean previousWasBlockStart = blockStart;
+ blockStart = content.isEmpty();
+
+ String open = openBlocks.peek();
+ if (open != null && closes(open, content)) {
+ openBlocks.pop();
+ if (isVerbatim(open)) {
+ verbatimBlocks.add(new int[] {verbatimStart, line.end()});
+ }
+ blockStart = true;
+ continue;
+ }
+ if (open != null && isVerbatim(open)) {
+ // Nothing nests inside a listing, literal, passthrough or comment
block.
+ continue;
+ }
+ if (previous != null && isSetextUnderline(previous.text(), content)) {
+ // The underline of a two-line title is not a block delimiter.
+ headings.add(
+ new Heading(
+ SETEXT_UNDERLINES.indexOf(content.charAt(0)) + 1,
+ previous.text().strip(),
+ previous.start(),
+ line.end()));
+ blockStart = true;
+ continue;
+ }
+ if (DELIMITER.matcher(content).matches()) {
+ if (documentTitle == null && !content.startsWith("////")) {
+ documentTitle = false;
+ }
+ if (isVerbatim(content)) {
+ verbatimStart = line.start();
+ }
+ openBlocks.push(content);
+ blockStart = true;
+ continue;
+ }
+ if (content.isEmpty()) {
+ continue;
+ }
+ if (content.startsWith("//")
+ || ATTRIBUTE_ENTRY.matcher(content).matches()
+ || (content.startsWith("[") && content.endsWith("]"))) {
+ // Comments, attribute entries and block attributes ([source], [[id]])
come before a block.
+ blockStart = previousWasBlockStart;
+ continue;
+ }
+ if (open != null) {
+ continue;
+ }
+ Matcher matcher = HEADING.matcher(line.text());
+ if (matcher.matches()) {
+ Heading heading =
+ new Heading(
+ matcher.group(1).length(),
+ stripClosingSequence(matcher.group(2).strip(),
matcher.group(1)),
+ line.start(),
+ line.end());
+ headings.add(heading);
+ oneLineTitles.add(heading);
+ if (documentTitle == null) {
+ documentTitle = heading.level() == 1;
+ }
+ blockStart = true;
+ continue;
+ }
+ if (documentTitle == null) {
+ documentTitle = false;
+ }
+ if (previousWasBlockStart && isSetextTitle(line.text())) {
+ titleCandidate = line;
+ }
+ }
+ return new Scan(
+ headings, oneLineTitles, verbatimBlocks, documentTitle != null &&
documentTitle);
+ }
+
+ /** A title line can not be indented or start with a period (a block title),
and has a word. */
+ private static boolean isSetextTitle(String line) {
+ return !line.isEmpty()
+ && !Character.isWhitespace(line.charAt(0))
+ && line.charAt(0) != '.'
+ && line.chars().anyMatch(Character::isLetterOrDigit);
+ }
+
+ /**
+ * An underline is one of {@code = - ~ ^ +} repeated, within one character
of the title length, as
+ * in Asciidoctor.
+ */
+ private static boolean isSetextUnderline(String title, String line) {
+ if (line.length() < 2 || SETEXT_UNDERLINES.indexOf(line.charAt(0)) < 0) {
+ return false;
+ }
+ char c = line.charAt(0);
+ return line.chars().allMatch(ch -> ch == c)
+ && Math.abs(title.stripTrailing().length() - line.length()) < 2;
+ }
+
+ private static boolean closes(String open, String line) {
+ return open.startsWith("```") ? line.equals("```") : line.equals(open);
+ }
+
+ private static boolean isVerbatim(String delimiter) {
+ char c = delimiter.charAt(0);
+ return delimiter.length() > 2 && (c == '-' || c == '.' || c == '+' || c ==
'/' || c == '`');
+ }
+
+ /** Drops the closing sequence of a symmetric title ({@code == Title ==}). */
+ private static String stripClosingSequence(String title, String marker) {
+ if (title.endsWith(marker)) {
+ String rest = title.substring(0, title.length() - marker.length());
+ if (rest.endsWith(" ") || rest.endsWith("\t")) {
+ return rest.strip();
+ }
+ }
+ return title;
}
}
diff --git
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistry.java
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistry.java
index 318ac8e427..74044f97e3 100644
---
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistry.java
+++
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistry.java
@@ -16,9 +16,11 @@
*/
package org.apache.hop.pipeline.transforms.chunker.document;
+import java.util.ArrayList;
import java.util.EnumMap;
+import java.util.List;
import java.util.Map;
-import java.util.regex.Pattern;
+import
org.apache.hop.pipeline.transforms.chunker.document.HeadingParserSupport.Heading;
import
org.apache.hop.pipeline.transforms.chunker.document.hopxml.HopPipelineXmlParser;
import
org.apache.hop.pipeline.transforms.chunker.document.hopxml.HopWorkflowXmlParser;
import
org.apache.hop.pipeline.transforms.chunker.document.hopxml.HopXmlSupport;
@@ -29,12 +31,6 @@ public final class DocumentParserRegistry {
private static final Map<ContentType, DocumentParser> PARSERS = new
EnumMap<>(ContentType.class);
- /** Must stay in sync with {@code AsciiDocDocumentParser.HEADING}. */
- private static final Pattern ASCIIDOC_HEADING = Pattern.compile("(?m)^=+[
\\t]+\\S");
-
- /** Must stay in sync with {@code MarkdownDocumentParser.HEADING}. */
- private static final Pattern MARKDOWN_HEADING =
Pattern.compile("(?m)^#{1,6}[ \\t]+\\S");
-
static {
PARSERS.put(ContentType.PLAIN, new PlainTextDocumentParser());
PARSERS.put(ContentType.MARKDOWN, new MarkdownDocumentParser());
@@ -56,26 +52,96 @@ public final class DocumentParserRegistry {
if (text == null || text.isBlank()) {
return ContentType.PLAIN;
}
- String trimmed = text.stripLeading();
- if (trimmed.startsWith("{") || text.contains("Configuration:\n{")) {
+ if (looksLikeJsonObject(text) || text.contains("Configuration:\n{")) {
return ContentType.METADATA;
}
- if (HopXmlSupport.looksLikeHopXml(text)) {
- if (trimmed.contains("<workflow")) {
- return ContentType.WORKFLOW;
- }
- if (trimmed.contains("<pipeline")) {
- return ContentType.PIPELINE;
- }
+ String rootElement = HopXmlSupport.rootElementName(text);
+ if ("workflow".equals(rootElement)) {
+ return ContentType.WORKFLOW;
+ }
+ if ("pipeline".equals(rootElement)) {
+ return ContentType.PIPELINE;
}
- // find() rather than matches(): a heading anywhere in the document
counts, including one on
- // the very first line, which is the common case for a single-page
document.
- if (ASCIIDOC_HEADING.matcher(text).find()) {
+ return detectHeadingSyntax(text);
+ }
+
+ /**
+ * Markdown or AsciiDoc, by the headings each parser would find. An AsciiDoc
document title
+ * ({@code = Title}) as the first content line settles it. Otherwise the
syntax with the most
+ * headings wins, and on a tie the syntax of the first heading. A {@code #}
line in an AsciiDoc
+ * verbatim block (a listing, comment, ...) does not count as a Markdown
heading, and a {@code =}
+ * line in a Markdown code fence does not count as an AsciiDoc title. A
single {@code =} line
+ * further down (an AsciiDoc document title) only counts when there are no
Markdown headings: next
+ * to them it is more likely text such as {@code = 42}. Two-line titles do
not count: Markdown has
+ * them too.
+ */
+ private static ContentType detectHeadingSyntax(String text) {
+ AsciiDocDocumentParser.Scan asciiDoc = AsciiDocDocumentParser.scan(text);
+ if (asciiDoc.startsWithDocumentTitle()) {
return ContentType.ASCIIDOC;
}
- if (MARKDOWN_HEADING.matcher(text).find()) {
+ MarkdownDocumentParser.Scan markdown = MarkdownDocumentParser.scan(text);
+ List<Heading> markdownHeadings = markdown.headings();
+ if (markdownHeadings.isEmpty()) {
+ return asciiDoc.oneLineTitles().isEmpty() ? ContentType.PLAIN :
ContentType.ASCIIDOC;
+ }
+ List<Heading> asciiDocSections =
+ asciiDoc.oneLineTitles().stream().filter(h -> h.level() > 1).toList();
+ if (asciiDocSections.isEmpty()) {
return ContentType.MARKDOWN;
}
- return ContentType.PLAIN;
+ // Both syntaxes have headings: the code blocks of each hide the headings
of the other.
+ markdownHeadings = outside(markdownHeadings, asciiDoc.verbatimBlocks());
+ asciiDocSections = outside(asciiDocSections, markdown.fencedBlocks());
+ if (asciiDocSections.size() != markdownHeadings.size()) {
+ return asciiDocSections.size() > markdownHeadings.size()
+ ? ContentType.ASCIIDOC
+ : ContentType.MARKDOWN;
+ }
+ return markdownHeadings.isEmpty()
+ || asciiDocSections.get(0).start() <
markdownHeadings.get(0).start()
+ ? ContentType.ASCIIDOC
+ : ContentType.MARKDOWN;
+ }
+
+ /** The headings that are not inside one of the {@code [start, end)} blocks;
both are sorted. */
+ private static List<Heading> outside(List<Heading> headings, List<int[]>
blocks) {
+ if (blocks.isEmpty()) {
+ return headings;
+ }
+ List<Heading> result = new ArrayList<>();
+ int b = 0;
+ for (Heading heading : headings) {
+ while (b < blocks.size() && blocks.get(b)[1] <= heading.start()) {
+ b++;
+ }
+ if (b == blocks.size() || heading.start() < blocks.get(b)[0]) {
+ result.add(heading);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * A JSON object: an opening brace followed by a quoted key or the closing
brace, and a closing
+ * brace at the end. A Hugo shortcode at the start of a Markdown page is not
one.
+ */
+ private static boolean looksLikeJsonObject(String text) {
+ int first = 0;
+ while (first < text.length() &&
Character.isWhitespace(text.charAt(first))) {
+ first++;
+ }
+ int last = text.length() - 1;
+ while (last > first && Character.isWhitespace(text.charAt(last))) {
+ last--;
+ }
+ if (last <= first || text.charAt(first) != '{' || text.charAt(last) !=
'}') {
+ return false;
+ }
+ int next = first + 1;
+ while (next < last && Character.isWhitespace(text.charAt(next))) {
+ next++;
+ }
+ return text.charAt(next) == '"' || next == last;
}
}
diff --git
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/HeadingParserSupport.java
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/HeadingParserSupport.java
index fa39b9815e..76dbc74f34 100644
---
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/HeadingParserSupport.java
+++
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/HeadingParserSupport.java
@@ -18,59 +18,74 @@ package org.apache.hop.pipeline.transforms.chunker.document;
import java.util.ArrayList;
import java.util.List;
-import java.util.function.ToIntFunction;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-/** Builds a {@link DocumentNode} tree from heading markers in Markdown or
AsciiDoc. */
+/** Builds a {@link DocumentNode} tree from the headings found in Markdown or
AsciiDoc. */
final class HeadingParserSupport {
+ /** Deepest heading level kept apart; deeper headings are nested at this
level. */
+ private static final int MAX_LEVEL = 6;
+
private HeadingParserSupport() {}
- static DocumentNode parse(String text, Pattern headingPattern,
ToIntFunction<String> levelFn) {
- if (text == null || text.isEmpty()) {
- return DocumentNode.root("");
+ /** A heading line: its level (1 = top), title and the offsets of the line
in the text. */
+ record Heading(int level, String title, int start, int end) {}
+
+ /** One line of the text without its line terminator, with its offsets in
the text. */
+ record Line(String text, int start, int end) {}
+
+ /** Splits text into lines on {@code \n}; a trailing {@code \r} is not part
of the line. */
+ static List<Line> lines(String text) {
+ List<Line> lines = new ArrayList<>();
+ int start = 0;
+ while (start <= text.length()) {
+ int newline = text.indexOf('\n', start);
+ int end = newline < 0 ? text.length() : newline;
+ int contentEnd = end > start && text.charAt(end - 1) == '\r' ? end - 1 :
end;
+ lines.add(new Line(text.substring(start, contentEnd), start,
contentEnd));
+ if (newline < 0) {
+ break;
+ }
+ start = newline + 1;
}
+ return lines;
+ }
- Matcher matcher = headingPattern.matcher(text);
- List<HeadingMatch> headings = new ArrayList<>();
- while (matcher.find()) {
- headings.add(
- new HeadingMatch(
- levelFn.applyAsInt(matcher.group(1)),
- matcher.group(2).trim(),
- matcher.start(),
- matcher.end()));
+ static DocumentNode parse(String text, List<Heading> headings) {
+ if (text == null || text.isEmpty()) {
+ return DocumentNode.root("");
}
-
if (headings.isEmpty()) {
return DocumentNode.root(text);
}
- String preamble = text.substring(0, headings.get(0).start).strip();
+ String preamble = text.substring(0, headings.get(0).start()).strip();
MutableNode root = new MutableNode("", preamble, 0, skipWhitespace(text,
0));
- MutableNode[] stack = new MutableNode[7];
+ MutableNode[] stack = new MutableNode[MAX_LEVEL + 1];
stack[0] = root;
for (int i = 0; i < headings.size(); i++) {
- HeadingMatch heading = headings.get(i);
- int bodyStart = heading.end;
- int bodyEnd = (i + 1 < headings.size()) ? headings.get(i + 1).start :
text.length();
+ Heading heading = headings.get(i);
+ int bodyStart = heading.end();
+ int bodyEnd = (i + 1 < headings.size()) ? headings.get(i + 1).start() :
text.length();
String body = text.substring(bodyStart, bodyEnd).strip();
- int level = Math.min(Math.max(heading.level, 1), stack.length - 1);
+ int level = Math.min(Math.max(heading.level(), 1), MAX_LEVEL);
MutableNode node =
- new MutableNode(heading.title, body, heading.start,
skipWhitespace(text, bodyStart));
+ new MutableNode(heading.title(), body, heading.start(),
skipWhitespace(text, bodyStart));
+ // The parent is the nearest open heading above this level, so a skipped
level
+ // (# Guide followed by ### Detail) still nests under Guide.
+ MutableNode parent = root;
+ for (int j = level - 1; j > 0; j--) {
+ if (stack[j] != null) {
+ parent = stack[j];
+ break;
+ }
+ }
+ parent.children.add(node);
stack[level] = node;
for (int j = level + 1; j < stack.length; j++) {
stack[j] = null;
}
- MutableNode parent = stack[level - 1];
- if (parent == null) {
- parent = stack[0];
- }
- parent.children.add(node);
- stack[level] = node;
}
return root.toImmutable();
@@ -85,20 +100,6 @@ final class HeadingParserSupport {
return result;
}
- private static final class HeadingMatch {
- private final int level;
- private final String title;
- private final int start;
- private final int end;
-
- private HeadingMatch(int level, String title, int start, int end) {
- this.level = level;
- this.title = title;
- this.start = start;
- this.end = end;
- }
- }
-
private static final class MutableNode {
private final String title;
private final String body;
diff --git
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/MarkdownDocumentParser.java
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/MarkdownDocumentParser.java
index e3adfea30e..bf7cc85004 100644
---
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/MarkdownDocumentParser.java
+++
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/MarkdownDocumentParser.java
@@ -16,12 +16,28 @@
*/
package org.apache.hop.pipeline.transforms.chunker.document;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import
org.apache.hop.pipeline.transforms.chunker.document.HeadingParserSupport.Heading;
+import
org.apache.hop.pipeline.transforms.chunker.document.HeadingParserSupport.Line;
-/** Splits Markdown on {@code #} headings and builds a section tree. */
+/**
+ * Splits Markdown on {@code #} headings and builds a section tree. Lines
inside a fenced code block
+ * ({@code ```} or {@code ~~~}) are never headings, so a shell comment in a
code sample stays in the
+ * section body.
+ */
public final class MarkdownDocumentParser implements DocumentParser {
- private static final Pattern HEADING =
Pattern.compile("(?m)^(#{1,6})\\s+(.+)$");
+ /** An ATX heading: 1-6 {@code #}, then a space or tab and the title, on a
single line. */
+ private static final Pattern HEADING = Pattern.compile("^(#{1,6})[
\\t]+(\\S.*)$");
+
+ /** The optional closing sequence of an ATX heading ({@code ## Install ##}).
*/
+ private static final Pattern CLOSING_SEQUENCE = Pattern.compile("(?:^|[
\\t]+)#+[ \\t]*$");
+
+ /** A code fence: up to 3 spaces of indentation, then 3 or more backticks or
tildes. */
+ private static final Pattern FENCE = Pattern.compile("^
{0,3}(`{3,}|~{3,})(.*)$");
@Override
public ContentType getContentType() {
@@ -30,6 +46,61 @@ public final class MarkdownDocumentParser implements
DocumentParser {
@Override
public DocumentNode parse(String text) {
- return HeadingParserSupport.parse(text, HEADING, marker ->
marker.length());
+ if (text == null || text.isEmpty()) {
+ return DocumentNode.root("");
+ }
+ return HeadingParserSupport.parse(text, scan(text).headings());
+ }
+
+ /**
+ * The headings of a Markdown text and the {@code [start, end)} offsets of
its fenced code blocks.
+ * An unclosed fence runs to the end of the text.
+ */
+ record Scan(List<Heading> headings, List<int[]> fencedBlocks) {}
+
+ static Scan scan(String text) {
+ List<Heading> headings = new ArrayList<>();
+ List<int[]> fencedBlocks = new ArrayList<>();
+ String openFence = null;
+ int fenceStart = -1;
+ for (Line line : HeadingParserSupport.lines(text)) {
+ Matcher fence = FENCE.matcher(line.text());
+ if (openFence != null) {
+ // A fence closes on the same character, at least as long, with
nothing after it.
+ if (fence.matches()
+ && fence.group(1).charAt(0) == openFence.charAt(0)
+ && fence.group(1).length() >= openFence.length()
+ && fence.group(2).isBlank()) {
+ openFence = null;
+ fencedBlocks.add(new int[] {fenceStart, line.end()});
+ }
+ continue;
+ }
+ if (fence.matches() && !(fence.group(1).charAt(0) == '`' &&
fence.group(2).contains("`"))) {
+ openFence = fence.group(1);
+ fenceStart = line.start();
+ continue;
+ }
+ Heading heading = heading(line);
+ if (heading != null) {
+ headings.add(heading);
+ }
+ }
+ if (openFence != null) {
+ fencedBlocks.add(new int[] {fenceStart, text.length()});
+ }
+ return new Scan(headings, fencedBlocks);
+ }
+
+ private static Heading heading(Line line) {
+ Matcher matcher = HEADING.matcher(line.text());
+ if (!matcher.matches()) {
+ return null;
+ }
+ String text = matcher.group(2).strip();
+ String title = CLOSING_SEQUENCE.matcher(text).replaceFirst("").strip();
+ // "## #" would otherwise get an empty title, so the breadcrumb ends in
"Guide > ".
+ return new Heading(
+ matcher.group(1).length(), title.isEmpty() ? text : title,
line.start(), line.end());
}
}
diff --git
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupport.java
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupport.java
index fc05aa6905..77f475b823 100644
---
a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupport.java
+++
b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupport.java
@@ -30,6 +30,7 @@ import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
/** DOM helpers for Hop .hpl / .hwf XML. */
public final class HopXmlSupport {
@@ -81,12 +82,98 @@ public final class HopXmlSupport {
private HopXmlSupport() {}
+ /** True when the text is an XML document whose root element is a pipeline
or a workflow. */
public static boolean looksLikeHopXml(String text) {
- if (text == null || text.isBlank()) {
- return false;
+ String root = rootElementName(text);
+ return "pipeline".equals(root) || "workflow".equals(root);
+ }
+
+ /**
+ * The name of the first element of an XML document, after the XML
declaration, comments and a
+ * document type declaration. Returns {@code null} when the text does not
start like XML, for
+ * example Markdown that mentions {@code <pipeline>} in a sentence.
+ */
+ public static String rootElementName(String text) {
+ if (text == null) {
+ return null;
+ }
+ // Walk the prolog by index: a .hpl or .hwf can be large and is checked
for every row.
+ int i = skipWhitespace(text, 0);
+ if (text.startsWith("\uFEFF", i)) {
+ i = skipWhitespace(text, i + 1);
+ }
+ while (text.startsWith("<", i)) {
+ int close;
+ if (text.startsWith("<?", i)) {
+ close = endOf(text, "?>", i);
+ } else if (text.startsWith("<!--", i)) {
+ close = endOf(text, "-->", i);
+ } else if (text.startsWith("<!", i)) {
+ close = endOfDeclaration(text, i);
+ } else {
+ int end = i + 1;
+ while (end < text.length() && isNameChar(text.charAt(end))) {
+ end++;
+ }
+ return end > i + 1 ? text.substring(i + 1, end) : null;
+ }
+ if (close < 0) {
+ return null;
+ }
+ i = skipWhitespace(text, close);
+ }
+ return null;
+ }
+
+ private static int skipWhitespace(String text, int from) {
+ int i = from;
+ while (i < text.length() && Character.isWhitespace(text.charAt(i))) {
+ i++;
}
- String trimmed = text.stripLeading();
- return trimmed.contains("<pipeline") || trimmed.contains("<workflow");
+ return i;
+ }
+
+ /** The offset just after the terminator, or -1. */
+ private static int endOf(String text, String terminator, int from) {
+ int index = text.indexOf(terminator, from);
+ return index < 0 ? -1 : index + terminator.length();
+ }
+
+ /**
+ * The offset just after a {@code <!DOCTYPE ...>} declaration, or -1. A
{@code >} inside quotes or
+ * inside the internal subset ({@code [ <!ENTITY ...> ]}) does not end it.
+ */
+ private static int endOfDeclaration(String text, int from) {
+ char quote = 0;
+ int depth = 0;
+ for (int i = from + 2; i < text.length(); i++) {
+ char c = text.charAt(i);
+ if (quote != 0) {
+ if (c == quote) {
+ quote = 0;
+ }
+ } else if (depth > 0 && text.startsWith("<!--", i)) {
+ // A comment in the internal subset can hold quotes and brackets.
+ int end = text.indexOf("-->", i + 4);
+ if (end < 0) {
+ return -1;
+ }
+ i = end + 2;
+ } else if (c == '"' || c == '\'') {
+ quote = c;
+ } else if (c == '[') {
+ depth++;
+ } else if (c == ']') {
+ depth--;
+ } else if (c == '>' && depth <= 0) {
+ return i + 1;
+ }
+ }
+ return -1;
+ }
+
+ private static boolean isNameChar(char c) {
+ return Character.isLetterOrDigit(c) || c == '_' || c == '-' || c == '.' ||
c == ':';
}
/**
@@ -144,6 +231,8 @@ public final class HopXmlSupport {
factory.setExpandEntityReferences(false);
factory.setNamespaceAware(false);
DocumentBuilder builder = factory.newDocumentBuilder();
+ // Without an error handler the parser prints "[Fatal Error] ..." to
stderr before throwing.
+ builder.setErrorHandler(new DefaultHandler());
return builder.parse(new InputSource(new StringReader(xml)));
} catch (Exception e) {
return null;
diff --git
a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistryTest.java
b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistryTest.java
index 1f55987594..90829466e9 100644
---
a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistryTest.java
+++
b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistryTest.java
@@ -90,4 +90,104 @@ class DocumentParserRegistryTest {
assertEquals(ContentType.MARKDOWN, parser.getContentType());
assertEquals(2, parser.parse(markdown).flattenSections().size());
}
+
+ @Test
+ void markdownMentioningHopXmlIsNotHopXml() {
+ String text = "# Pipelines\n\nA <pipeline> runs transforms, a <workflow>
runs actions.";
+ assertEquals(ContentType.MARKDOWN, DocumentParserRegistry.detect(text));
+ }
+
+ @Test
+ void detectsHopXmlAfterTheDeclarationAndLicenseComment() {
+ String header = "<?xml version=\"1.0\"
encoding=\"UTF-8\"?>\n<!--\nlicense\n-->\n";
+ assertEquals(
+ ContentType.PIPELINE,
+ DocumentParserRegistry.detect(header +
"<pipeline><info/><workflow_name/></pipeline>"));
+ assertEquals(
+ ContentType.WORKFLOW,
+ DocumentParserRegistry.detect(header +
"<workflow><name/></workflow>"));
+ }
+
+ @Test
+ void markdownStartingWithAShortcodeIsNotJson() {
+ String text = "{{< note >}}\nRead this first.\n{{< /note >}}\n\n#
Guide\n\nIntro.";
+ assertEquals(ContentType.MARKDOWN, DocumentParserRegistry.detect(text));
+ }
+
+ @Test
+ void detectsMetadataJson() {
+ assertEquals(
+ ContentType.METADATA, DocumentParserRegistry.detect(" {\n \"name\":
\"conn\"\n}\n"));
+ }
+
+ @Test
+ void anEqualsLineInMarkdownDoesNotMakeItAsciiDoc() {
+ String text = "# Guide\n\nThe answer:\n= 42\n\n## Next\n\nMore text.";
+ assertEquals(ContentType.MARKDOWN, DocumentParserRegistry.detect(text));
+ }
+
+ @Test
+ void asciiDocPageWithShellCommentsIsAsciiDoc() {
+ String page =
+ "////\nLicensed to the ASF\n////\n:documentationPath:
/pipeline/\n:language: en_US\n\n"
+ + "[[Install]]\n= Install\n\n== Download\n\n[source,bash]\n----\n#
download\n"
+ + "# unzip\n# run\n----\n";
+ assertEquals(ContentType.ASCIIDOC, DocumentParserRegistry.detect(page));
+ // Without the document title the shell comments in the listing block
still do not count.
+ String fragment = page.substring(page.indexOf("== Download"));
+ assertEquals(ContentType.ASCIIDOC,
DocumentParserRegistry.detect(fragment));
+ }
+
+ @Test
+ void markdownWithCodeFenceCommentsIsMarkdown() {
+ String text = "Intro\n\n## Install\n\n```\n# a\n# b\n```\n";
+ assertEquals(ContentType.MARKDOWN, DocumentParserRegistry.detect(text));
+ }
+
+ @Test
+ void hashLinesInAsciiDocCommentAndPassthroughBlocksAreNotMarkdownHeadings() {
+ assertEquals(
+ ContentType.ASCIIDOC,
+ DocumentParserRegistry.detect("== Setup\n\nText.\n\n////\n# one\n#
two\n////\n"));
+ assertEquals(
+ ContentType.ASCIIDOC,
+ DocumentParserRegistry.detect("== Setup\n\nText.\n\n++++\n# one\n#
two\n++++\n"));
+ }
+
+ @Test
+ void asciiDocTitlesInAMarkdownTildeFenceDoNotCount() {
+ String text = "## Example\n\n~~~\n== Section A\n\n== Section B\n~~~\n";
+ assertEquals(ContentType.MARKDOWN, DocumentParserRegistry.detect(text));
+ }
+
+ @Test
+ void onATieTheFirstHeadingDecides() {
+ assertEquals(
+ ContentType.MARKDOWN,
+ DocumentParserRegistry.detect("# Guide\n\nBody.\n\n== Note:
tie-break\n\nMore."));
+ assertEquals(
+ ContentType.ASCIIDOC,
+ DocumentParserRegistry.detect("== Download\n\nBody.\n\n# not a
heading\n\nMore."));
+ }
+
+ @Test
+ void aDocumentTitleAfterANestedCommentDelimiterIsFound() {
+ // Inside a ////// comment a //// line is text, it does not close the
comment.
+ String text = "//////\n////\n# commented\n//////\n= Title\n\n# one\n\n#
two\n";
+ assertEquals(ContentType.ASCIIDOC, DocumentParserRegistry.detect(text));
+ }
+
+ @Test
+ void markdownTwoLineTitlesDoNotMakeItAsciiDoc() {
+ String text = "Guide\n=====\n\nIntro.\n\nInstall\n-------\n\nSteps.\n\n##
Detail\n";
+ assertEquals(ContentType.MARKDOWN, DocumentParserRegistry.detect(text));
+ }
+
+ @Test
+ void detectsHopXmlWithADoctypeInternalSubset() {
+ String xml =
+ "<?xml version=\"1.0\"?>\n<!DOCTYPE pipeline [\n <!ENTITY name
\"x>y\">\n"
+ + " <!-- it's ] here -->\n]>\n<pipeline><info/></pipeline>";
+ assertEquals(ContentType.PIPELINE, DocumentParserRegistry.detect(xml));
+ }
}
diff --git
a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/HeadingParsersTest.java
b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/HeadingParsersTest.java
new file mode 100644
index 0000000000..5a1d9de520
--- /dev/null
+++
b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/HeadingParsersTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.List;
+import org.junit.jupiter.api.Test;
+
+/** Markdown and AsciiDoc section parsing (#9282). */
+class HeadingParsersTest {
+
+ private static List<String> sections(DocumentParser parser, String text) {
+ return parser.parse(text).flattenSections().stream()
+ .map(s -> s.breadcrumb() + ": " + s.getBody())
+ .toList();
+ }
+
+ @Test
+ void asciiDocExampleBlockDelimiterIsNotAHeading() {
+ String text =
+ "= Title\n\nIntro.\n\n== Section\n\nBefore the block.\n\n====\n"
+ + "Example text inside the block.\n====\n\nAfter the block.\n\n==
Next\n\nMore text.";
+
+ assertEquals(
+ List.of(
+ "Title: Intro.",
+ "Title > Section: Before the block.\n\n====\nExample text inside
the block.\n====\n\n"
+ + "After the block.",
+ "Title > Next: More text."),
+ sections(new AsciiDocDocumentParser(), text));
+ }
+
+ @Test
+ void asciiDocTitlesInsideDelimitedBlocksAreIgnored() {
+ String text =
+ "= Title\n\nIntro.\n\n[source]\n----\n== not a section\n----\n\n"
+ + "////\n== commented out\n////\n\n"
+ + "====\n.Nested\n------\n= neither\n------\n====\n\n==
Next\n\nMore text.";
+
+ List<String> sections = sections(new AsciiDocDocumentParser(), text);
+
+ assertEquals(2, sections.size());
+ assertEquals("Title > Next: More text.", sections.get(1));
+ }
+
+ @Test
+ void asciiDocSymmetricTitleDropsTheClosingSequence() {
+ assertEquals(
+ List.of("Install: Text.", "Install > Install = Upgrade: More."),
+ sections(
+ new AsciiDocDocumentParser(),
+ "== Install ==\n\nText.\n\n=== Install = Upgrade ===\n\nMore."));
+ }
+
+ @Test
+ void asciiDocDoesNotTreatAnEmptyHeadingMarkerAsAHeading() {
+ // The old pattern let \s+ match the line break, so a bare "==" took the
next line as title.
+ assertEquals(
+ List.of("Title: Intro.\n==\nnot a title"),
+ sections(new AsciiDocDocumentParser(), "= Title\nIntro.\n==\nnot a
title"));
+ }
+
+ @Test
+ void markdownCodeFenceLinesAreNotHeadings() {
+ String text =
+ "# Guide\n\nIntro.\n\n```bash\n# download the zip\ncurl -O
https://example.com/hop.zip\n```"
+ + "\n\nAfter the block.\n\n~~~~\n## also code\n```\n~~~~\n\n##
Next\n\nMore.";
+
+ assertEquals(
+ List.of(
+ "Guide: Intro.\n\n```bash\n# download the zip\ncurl -O
https://example.com/hop.zip\n```"
+ + "\n\nAfter the block.\n\n~~~~\n## also code\n```\n~~~~",
+ "Guide > Next: More."),
+ sections(new MarkdownDocumentParser(), text));
+ }
+
+ @Test
+ void markdownUnclosedFenceRunsToTheEnd() {
+ assertEquals(
+ List.of("Guide: ```\n# comment"),
+ sections(new MarkdownDocumentParser(), "# Guide\n```\n# comment"));
+ }
+
+ @Test
+ void markdownSkippedLevelKeepsTheNearestAncestor() {
+ assertEquals(
+ List.of("Guide: Intro.", "Guide > Detail: Detail text.", "Guide >
Section: Body."),
+ sections(
+ new MarkdownDocumentParser(),
+ "# Guide\n\nIntro.\n\n### Detail\n\nDetail text.\n\n##
Section\n\nBody."));
+ }
+
+ @Test
+ void markdownClosingHashesAreNotPartOfTheTitle() {
+ assertEquals(
+ List.of("Guide: Intro.", "Guide > Install: Text.", "Guide > Install >
C#: Code."),
+ sections(
+ new MarkdownDocumentParser(),
+ "# Guide #\n\nIntro.\n\n## Install ## \n\nText.\n\n###
C#\n\nCode."));
+ }
+
+ @Test
+ void markdownHashWithoutTitleIsNotAHeading() {
+ assertEquals(
+ List.of("Guide: #\nnot a title"),
+ sections(new MarkdownDocumentParser(), "# Guide\n#\nnot a title"));
+ }
+
+ @Test
+ void windowsLineEndingsAreHandled() {
+ assertEquals(
+ List.of("Guide: Intro.", "Guide > Next: More."),
+ sections(
+ new MarkdownDocumentParser(), "# Guide\r\n\r\nIntro.\r\n\r\n##
Next ##\r\n\r\nMore."));
+ }
+
+ @Test
+ void asciiDocTwoLineTitleUnderlineIsNotABlockDelimiter() {
+ assertEquals(
+ List.of("Introduction: Text.", "Introduction > Install: Steps."),
+ sections(
+ new AsciiDocDocumentParser(),
+ "Introduction\n============\n\nText.\n\n== Install\n\nSteps."));
+ assertEquals(
+ List.of("Setup: Text.", "Setup > Install: Steps."),
+ sections(new AsciiDocDocumentParser(), "Setup\n-----\n\nText.\n\n===
Install\n\nSteps."));
+ }
+
+ @Test
+ void asciiDocDelimiterAfterABlankLineOrOfAnotherLengthOpensABlock() {
+ // After a blank line, or not within one character of the line above, ----
opens a listing.
+ assertEquals(
+ List.of("Title: Text\n\n----\n== code\n----", "Title > Next: More."),
+ sections(
+ new AsciiDocDocumentParser(),
+ "= Title\n\nText\n\n----\n== code\n----\n\n== Next\n\nMore."));
+ assertEquals(
+ List.of("Title: A longer paragraph line\n----\n== code\n----", "Title
> Next: More."),
+ sections(
+ new AsciiDocDocumentParser(),
+ "= Title\n\nA longer paragraph line\n----\n== code\n----\n\n==
Next\n\nMore."));
+ }
+
+ @Test
+ void markdownClosingSequenceOnlyKeepsTheText() {
+ assertEquals(
+ List.of("Guide: Intro.", "Guide > #: Text.", "Guide > # > ###: More."),
+ sections(
+ new MarkdownDocumentParser(),
+ "# Guide\n\nIntro.\n\n## #\n\nText.\n\n### ###\n\nMore."));
+ }
+}