This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-mcp.git
The following commit(s) were added to refs/heads/main by this push:
new 20e84a7 feat(indexing): add markdown document indexing (#144)
20e84a7 is described below
commit 20e84a7d33b92970b40e1c3a7ad810c6fb534e7b
Author: Shahzad ASGHAR [United Nations] <[email protected]>
AuthorDate: Fri Sep 11 17:10:25 2026 +0300
feat(indexing): add markdown document indexing (#144)
* feat(indexing): add markdown document indexing
Adds an index-markdown-documents MCP tool that indexes markdown
content into Solr, complementing the existing JSON/CSV/XML tools.
- New MarkdownDocumentCreator with CommonMark and YAML front matter
- Stable SHA-256 content IDs when front matter has no id
- Multi-valued heading and YAML list fields
- Markdown support in the index-data prompt
- Unit and integration coverage
Refs #69
* feat(indexing): steer clients away from markdown tool for structured
formats
Direct clients to the dedicated JSON, CSV, and XML tools. Convert to
Markdown only when no dedicated source-format tool exists, and require
a stable front matter id for converted content.
---
AGENTS.md | 4 +-
README.md | 1 +
build.gradle.kts | 3 +
gradle/libs.versions.toml | 5 +
.../solr/mcp/server/indexing/IndexingService.java | 93 ++++++-
.../documentcreator/IndexingDocumentCreator.java | 36 ++-
.../documentcreator/MarkdownDocumentCreator.java | 281 ++++++++++++++++++++
.../mcp/server/McpClientIntegrationTestBase.java | 34 +++
.../indexing/IndexingServiceIntegrationTest.java | 4 +-
.../mcp/server/indexing/IndexingServiceTest.java | 16 ++
.../mcp/server/indexing/MarkdownIndexingTest.java | 287 +++++++++++++++++++++
11 files changed, 753 insertions(+), 11 deletions(-)
diff --git a/AGENTS.md b/AGENTS.md
index 1f4cd3a..89db734 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -99,7 +99,7 @@ docker run -p 8080:8080 --rm -e PROFILES=http \
Four service classes expose MCP tools via `@McpTool` annotations:
- **SearchService** (`search/`) - Full-text search with filtering, faceting,
sorting, pagination
-- **IndexingService** (`indexing/`) - Document indexing supporting JSON, CSV,
XML formats
+- **IndexingService** (`indexing/`) - Document indexing supporting JSON, CSV,
XML, and markdown formats
- **CollectionService** (`collection/`) - List collections, get stats, health
checks
- **SchemaService** (`schema/`) - Schema introspection and additive
modification (add-fields, add-field-types)
@@ -107,7 +107,7 @@ Four service classes expose MCP tools via `@McpTool`
annotations:
`indexing/documentcreator/` uses strategy pattern for format parsing:
- `SolrDocumentCreator` - Common interface
-- `JsonDocumentCreator`, `CsvDocumentCreator`, `XmlDocumentCreator` - Format
implementations
+- `JsonDocumentCreator`, `CsvDocumentCreator`, `XmlDocumentCreator`,
`MarkdownDocumentCreator` - Format implementations
- `IndexingDocumentCreator` - Orchestrator that delegates to format-specific
creators
- `FieldNameSanitizer` - Automatic field name validation for Solr compatibility
diff --git a/README.md b/README.md
index ec3c5e1..0d0c1f7 100644
--- a/README.md
+++ b/README.md
@@ -99,6 +99,7 @@ Using a different client, or want STDIO/HTTP/Docker options?
See the per-client
| `index-json-documents` | Index documents from a JSON string into a
collection |
| `index-csv-documents` | Index documents from a CSV string into a collection |
| `index-xml-documents` | Index documents from an XML string into a collection
|
+| `index-markdown-documents` | Index a markdown document into a collection,
extracting front matter, title, headings, and body text |
| `create-collection` | Create a collection (configSet, numShards,
replicationFactor optional — default `_default`, `1`, `1`) |
| `list-collections` | List all available Solr collections |
| `get-collection-stats` | Get statistics and metrics for a collection |
diff --git a/build.gradle.kts b/build.gradle.kts
index 6a846c9..094209e 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -151,6 +151,9 @@ dependencies {
implementation(libs.spring.ai.starter.mcp.server.webmvc)
implementation(libs.solr.solrj)
implementation(libs.commons.csv)
+ // CommonMark for markdown parsing
+ implementation(libs.commonmark)
+ implementation(libs.commonmark.ext.yaml.front.matter)
// JSpecify for nullability annotations
implementation(libs.jspecify)
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 473ff9f..b412931 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -28,6 +28,7 @@ cyclonedx-plugin = "2.4.1"
spring-ai = "1.1.7"
solr = "10.0.0"
commons-csv = "1.14.1"
+commonmark = "0.28.0"
jspecify = "1.0.0"
mcp-server-security = "0.0.6"
@@ -66,6 +67,10 @@ solr-solrj = { module = "org.apache.solr:solr-solrj",
version.ref = "solr" }
# Apache Commons
commons-csv = { module = "org.apache.commons:commons-csv", version.ref =
"commons-csv" }
+# CommonMark (markdown parsing)
+commonmark = { module = "org.commonmark:commonmark", version.ref =
"commonmark" }
+commonmark-ext-yaml-front-matter = { module =
"org.commonmark:commonmark-ext-yaml-front-matter", version.ref = "commonmark" }
+
# Null safety
jspecify = { module = "org.jspecify:jspecify", version.ref = "jspecify" }
diff --git
a/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java
b/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java
index 3467485..5ac3704 100644
--- a/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java
+++ b/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java
@@ -42,11 +42,11 @@ import org.xml.sax.SAXException;
* Apache Solr collections through Model Context Protocol (MCP) integration.
*
* <p>
- * This service handles the conversion of JSON, CSV, and XML documents into
- * Solr-compatible format and manages the indexing process with robust error
- * handling and batch processing capabilities. It employs a schema-less
approach
- * where Solr automatically detects field types, eliminating the need for
- * predefined schema configuration.
+ * This service handles the conversion of JSON, CSV, XML, and markdown
documents
+ * into Solr-compatible format and manages the indexing process with robust
+ * error handling and batch processing capabilities. It employs a schema-less
+ * approach where Solr automatically detects field types, eliminating the need
+ * for predefined schema configuration.
*
* <p>
* <strong>Core Features:</strong>
@@ -60,6 +60,8 @@ import org.xml.sax.SAXException;
* with headers
* <li><strong>XML Processing</strong>: Support for XML documents with element
* flattening and attribute handling
+ * <li><strong>Markdown Processing</strong>: Support for markdown documents
with
+ * front matter, title, and heading extraction
* <li><strong>Batch Processing</strong>: Efficient bulk indexing with
* configurable batch sizes
* <li><strong>Error Resilience</strong>: Individual document fallback when
@@ -393,6 +395,81 @@ public class IndexingService {
+ collection + "'" +
describeIndexedFields(schemalessDoc);
}
+ /**
+ * Indexes a document from a markdown string into a specified Solr
collection.
+ *
+ * <p>
+ * This method serves as the primary entry point for markdown document
indexing
+ * operations and is exposed as an MCP tool for AI client interactions.
Unlike
+ * the structured formats (JSON, CSV, XML), markdown is a prose format,
so
+ * searchable structure is extracted from the document content itself.
+ *
+ * <p>
+ * <strong>Field Extraction:</strong>
+ *
+ * <ul>
+ * <li><strong>YAML Front Matter</strong>: Each entry becomes a
document field
+ * with a sanitized name (multi-valued where applicable)
+ * <li><strong>title</strong>: From the {@code title} front matter
entry, or the
+ * first level-1 heading
+ * <li><strong>headings</strong>: Multi-valued field with the text of
every
+ * heading (the document outline)
+ * <li><strong>content</strong>: Plain text body for full-text search
(front
+ * matter excluded)
+ * </ul>
+ *
+ * <p>
+ * <strong>MCP Tool Usage:</strong>
+ *
+ * <p>
+ * AI clients can invoke this method with natural language requests
like "index
+ * this markdown file into my_collection" or "add this README to the
search
+ * index".
+ *
+ * <p>
+ * <strong>Example Markdown Processing:</strong>
+ *
+ * <pre>{@code
+ * Input:
+ * ---
+ * author: Jane Doe
+ * ---
+ * # Getting Started
+ * Run the installer.
+ *
+ * Result: {author:"Jane Doe", title:"Getting Started",
+ * headings:["Getting Started"], content:"Getting Started\nRun
the installer."}
+ * }</pre>
+ *
+ * @param collection
+ * the name of the Solr collection to index documents into
+ * @param markdown
+ * markdown string to index, optionally starting with YAML
front
+ * matter
+ * @throws IOException
+ * if there are critical errors in Solr communication
+ * @throws SolrServerException
+ * if Solr server encounters errors during indexing
+ * @see
IndexingDocumentCreator#createSchemalessDocumentsFromMarkdown(String)
+ * @see #indexDocuments(String, List)
+ */
+ @PreAuthorize("isAuthenticated()")
+ @McpTool(
+ name = "index-markdown-documents",
+ annotations = @McpTool.McpAnnotations(idempotentHint =
true),
+ description = "Index a document from markdown String
into Solr collection, extracting front matter, title, headings, and body text. "
+ + "Do NOT use for JSON/CSV/XML input;
use index-json-documents, index-csv-documents, or index-xml-documents instead. "
+ + "Only convert source content to
markdown when there is no dedicated tool for the source format, and supply a
stable 'id' in the YAML front matter when doing so.")
+ public String indexMarkdownDocuments(@McpToolParam(description = "Solr
collection to index into") String collection,
+ @McpToolParam(
+ description = "Markdown string to
index, optionally starting with YAML front matter") String markdown)
+ throws IOException, SolrServerException {
+ List<SolrInputDocument> schemalessDoc =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+ int successCount = indexDocuments(collection, schemalessDoc);
+ return "Successfully indexed " + successCount + " of " +
schemalessDoc.size() + " documents into collection '"
+ + collection + "'";
+ }
+
/**
* Indexes a list of SolrInputDocument objects into a Solr collection
using
* batch processing.
@@ -532,7 +609,9 @@ public class IndexingService {
case "json" -> new IndexTool("index-json-documents",
"json");
case "csv" -> new IndexTool("index-csv-documents",
"csv");
case "xml" -> new IndexTool("index-xml-documents",
"xml");
- default -> throw new IllegalArgumentException("format
must be one of json/csv/xml, got: " + format);
+ case "markdown", "md" -> new
IndexTool("index-markdown-documents", "markdown");
+ default ->
+ throw new IllegalArgumentException("format must
be one of json/csv/xml/markdown, got: " + format);
};
}
@@ -563,7 +642,7 @@ public class IndexingService {
required = true) String collection,
@McpArg(
name = "format",
- description = "Document format: 'json',
'csv', or 'xml'",
+ description = "Document format: 'json',
'csv', 'xml', or 'markdown'",
required = true) String format,
@McpArg(
name = "sample",
diff --git
a/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/IndexingDocumentCreator.java
b/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/IndexingDocumentCreator.java
index 31e3b91..d180171 100644
---
a/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/IndexingDocumentCreator.java
+++
b/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/IndexingDocumentCreator.java
@@ -44,6 +44,8 @@ import org.springframework.stereotype.Service;
* with headers
* <li><strong>XML Processing</strong>: Support for XML documents with element
* flattening and attribute handling
+ * <li><strong>Markdown Processing</strong>: Support for markdown documents
with
+ * front matter, title, and heading extraction
* <li><strong>Field Sanitization</strong>: Automatic cleanup of field names
for
* Solr compatibility
* </ul>
@@ -62,6 +64,8 @@ public class IndexingDocumentCreator {
private final JsonDocumentCreator jsonDocumentCreator;
+ private final MarkdownDocumentCreator markdownDocumentCreator;
+
/**
* Constructs the orchestrator with the per-format document creators.
*
@@ -71,12 +75,15 @@ public class IndexingDocumentCreator {
* converts CSV input into {@code SolrInputDocument} batches
* @param jsonDocumentCreator
* converts JSON input into {@code SolrInputDocument} batches
+ * @param markdownDocumentCreator
+ * converts Markdown input into {@code SolrInputDocument}
batches
*/
public IndexingDocumentCreator(XmlDocumentCreator xmlDocumentCreator,
CsvDocumentCreator csvDocumentCreator,
- JsonDocumentCreator jsonDocumentCreator) {
+ JsonDocumentCreator jsonDocumentCreator,
MarkdownDocumentCreator markdownDocumentCreator) {
this.xmlDocumentCreator = xmlDocumentCreator;
this.csvDocumentCreator = csvDocumentCreator;
this.jsonDocumentCreator = jsonDocumentCreator;
+ this.markdownDocumentCreator = markdownDocumentCreator;
}
/**
@@ -144,4 +151,31 @@ public class IndexingDocumentCreator {
return xmlDocumentCreator.create(xml);
}
+
+ /**
+ * Creates a list of schema-less SolrInputDocument objects from a
markdown
+ * string.
+ *
+ * <p>
+ * This method delegates markdown processing to the
MarkdownDocumentCreator
+ * utility class.
+ *
+ * @param markdown
+ * markdown string containing document content, optionally
starting
+ * with YAML front matter
+ * @return list of SolrInputDocument objects ready for indexing
+ * @throws DocumentProcessingException
+ * if markdown parsing fails or input validation fails
+ * @see MarkdownDocumentCreator
+ */
+ public List<SolrInputDocument>
createSchemalessDocumentsFromMarkdown(String markdown)
+ throws DocumentProcessingException {
+
+ // Input validation
+ if (markdown == null || markdown.trim().isEmpty()) {
+ throw new DocumentProcessingException("Markdown input
cannot be null or empty");
+ }
+
+ return markdownDocumentCreator.create(markdown);
+ }
}
diff --git
a/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/MarkdownDocumentCreator.java
b/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/MarkdownDocumentCreator.java
new file mode 100644
index 0000000..db56020
--- /dev/null
+++
b/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/MarkdownDocumentCreator.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.mcp.server.indexing.documentcreator;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.HexFormat;
+import java.util.List;
+import org.apache.solr.common.SolrInputDocument;
+import org.commonmark.Extension;
+import org.commonmark.ext.front.matter.YamlFrontMatterExtension;
+import org.commonmark.ext.front.matter.YamlFrontMatterVisitor;
+import org.commonmark.node.AbstractVisitor;
+import org.commonmark.node.Code;
+import org.commonmark.node.CustomBlock;
+import org.commonmark.node.Heading;
+import org.commonmark.node.Node;
+import org.commonmark.node.Text;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.text.TextContentRenderer;
+import org.jspecify.annotations.Nullable;
+import org.springframework.stereotype.Component;
+
+/**
+ * Utility class for processing markdown documents and converting them to
+ * SolrInputDocument objects.
+ *
+ * <p>
+ * Unlike the structured formats (JSON, CSV, XML), markdown is a prose format,
+ * so this creator extracts searchable structure from the document rather than
+ * mapping fields one-to-one. Parsing is performed with the CommonMark library,
+ * which is lightweight and reflection-free (GraalVM native-image safe).
+ *
+ * <p>
+ * <strong>Field Extraction Rules:</strong>
+ *
+ * <ul>
+ * <li><strong>YAML Front Matter</strong>: Each front matter entry becomes a
+ * document field with a sanitized name. Entries with multiple values become
+ * multi-valued fields.
+ * <li><strong>id</strong>: Taken from the {@code id} front matter entry when
+ * present, otherwise derived deterministically from a SHA-256 hash of the
input
+ * so that re-indexing the same markdown overwrites the same document (keeping
+ * the operation idempotent).
+ * <li><strong>title</strong>: Taken from the {@code title} front matter entry
+ * when present, otherwise from the first level-1 heading.
+ * <li><strong>headings</strong>: Multi-valued field containing the text of
+ * every heading, preserving the document outline for searching.
+ * <li><strong>content</strong>: The plain text of the document body (front
+ * matter excluded), suitable for full-text search.
+ * </ul>
+ *
+ * <p>
+ * <strong>Example Transformation:</strong>
+ *
+ * <pre>{@code
+ * Input markdown:
+ * ---
+ * author: Jane Doe
+ * tags: [search, solr]
+ * ---
+ * # Getting Started
+ * ## Installation
+ * Run the installer.
+ *
+ * Output document:
+ * {author:"Jane Doe", tags:["search","solr"], title:"Getting Started",
+ * headings:["Getting Started","Installation"], content:"Getting
Started\nInstallation\nRun the installer."}
+ * }</pre>
+ *
+ * @see SolrInputDocument
+ * @see FieldNameSanitizer#sanitizeFieldName(String)
+ */
+@Component
+public class MarkdownDocumentCreator implements SolrDocumentCreator {
+
+ private static final int MAX_INPUT_SIZE_BYTES = 10 * 1024 * 1024;
+
+ /** Solr field holding the document's unique key. */
+ public static final String FIELD_ID = "id";
+
+ /** Solr field holding the document title. */
+ public static final String FIELD_TITLE = "title";
+
+ /** Multi-valued Solr field holding the text of every heading. */
+ public static final String FIELD_HEADINGS = "headings";
+
+ /** Solr field holding the plain text body of the document. */
+ public static final String FIELD_CONTENT = "content";
+
+ private final Parser parser;
+
+ private final TextContentRenderer textContentRenderer;
+
+ public MarkdownDocumentCreator() {
+ List<Extension> extensions =
List.of(YamlFrontMatterExtension.create());
+ this.parser = Parser.builder().extensions(extensions).build();
+ this.textContentRenderer =
TextContentRenderer.builder().build();
+ }
+
+ /**
+ * Creates a SolrInputDocument from a markdown string.
+ *
+ * <p>
+ * The whole input is treated as a single document: front matter
entries map to
+ * fields, the title is resolved from front matter or the first level-1
heading,
+ * all heading texts are collected into a multi-valued {@code headings}
field,
+ * and the plain text body is stored in {@code content}.
+ *
+ * @param markdown
+ * markdown string, optionally starting with YAML front
matter
+ * @return a single-element list containing the created document, or an
empty
+ * list if the input is blank
+ * @throws DocumentProcessingException
+ * if the input exceeds the size limit or parsing fails
+ */
+ @Override
+ public List<SolrInputDocument> create(String markdown) throws
DocumentProcessingException {
+ if (markdown.getBytes(StandardCharsets.UTF_8).length >
MAX_INPUT_SIZE_BYTES) {
+ throw new DocumentProcessingException(
+ "Input too large: exceeds maximum size
of " + MAX_INPUT_SIZE_BYTES + " bytes");
+ }
+
+ if (markdown.trim().isEmpty()) {
+ return List.of();
+ }
+
+ Node document;
+ try {
+ document = parser.parse(markdown);
+ } catch (RuntimeException e) {
+ throw new DocumentProcessingException("Failed to parse
markdown document", e);
+ }
+
+ SolrInputDocument doc = new SolrInputDocument();
+
+ addFrontMatterFields(document, doc);
+
+ // Solr's default schema requires a unique key. A
content-derived id keeps
+ // re-indexing of the same markdown idempotent (same input,
same document)
+ if (doc.getFieldValue(FIELD_ID) == null) {
+ doc.addField(FIELD_ID, contentHash(markdown));
+ }
+
+ HeadingCollector headingCollector = new HeadingCollector();
+ document.accept(headingCollector);
+ for (String heading : headingCollector.headings) {
+ doc.addField(FIELD_HEADINGS, heading);
+ }
+
+ // Front matter title wins; otherwise fall back to the first
level-1 heading
+ if (doc.getFieldValue(FIELD_TITLE) == null &&
headingCollector.firstTopLevelHeading != null) {
+ doc.addField(FIELD_TITLE,
headingCollector.firstTopLevelHeading);
+ }
+
+ String content = textContentRenderer.render(document).trim();
+ if (!content.isEmpty()) {
+ doc.addField(FIELD_CONTENT, content);
+ }
+
+ return List.of(doc);
+ }
+
+ /**
+ * Extracts YAML front matter entries into document fields and unlinks
the front
+ * matter block so it is excluded from the rendered body content.
+ */
+ private void addFrontMatterFields(Node document, SolrInputDocument doc)
{
+ YamlFrontMatterVisitor frontMatterVisitor = new
YamlFrontMatterVisitor();
+ document.accept(frontMatterVisitor);
+
+ frontMatterVisitor.getData().forEach((key, values) -> {
+ String fieldName =
FieldNameSanitizer.sanitizeFieldName(key);
+ for (String value : flattenFlowSequences(values)) {
+ if (!value.isEmpty()) {
+ doc.addField(fieldName, value);
+ }
+ }
+ });
+
+ // The front matter block is metadata, not body text: remove it
so the
+ // TextContentRenderer output contains only the document body
+ Node firstChild = document.getFirstChild();
+ if (firstChild instanceof CustomBlock) {
+ firstChild.unlink();
+ }
+ }
+
+ /**
+ * Expands simple YAML flow sequences into individual values.
+ *
+ * <p>
+ * The CommonMark front matter extension parses block-style lists
+ * ({@code - item}) into multiple values but passes flow-style lists
+ * ({@code [a, b, c]}) through as a single literal string. Flow style
is common
+ * for tags in real-world markdown (Jekyll, Hugo), so split it here to
produce
+ * the same multi-valued field either way. Values containing commas
inside
+ * quotes are not supported and are kept as-is.
+ */
+ private static List<String> flattenFlowSequences(List<String> values) {
+ List<String> result = new ArrayList<>(values.size());
+ for (String value : values) {
+ String trimmed = value.trim();
+ if (trimmed.length() >= 2 && trimmed.startsWith("[") &&
trimmed.endsWith("]") && !trimmed.contains("\"")
+ && !trimmed.contains("'")) {
+ for (String element : trimmed.substring(1,
trimmed.length() - 1).split(",")) {
+ result.add(element.trim());
+ }
+ } else {
+ result.add(value);
+ }
+ }
+ return result;
+ }
+
+ private static String contentHash(String markdown) {
+ try {
+ MessageDigest digest =
MessageDigest.getInstance("SHA-256");
+ return
HexFormat.of().formatHex(digest.digest(markdown.getBytes(StandardCharsets.UTF_8)));
+ } catch (NoSuchAlgorithmException e) {
+ // SHA-256 is guaranteed to be available on every Java
platform
+ throw new IllegalStateException("SHA-256 MessageDigest
not available", e);
+ }
+ }
+
+ /**
+ * AST visitor collecting heading texts and the first level-1 heading
for use as
+ * a title fallback.
+ */
+ private static final class HeadingCollector extends AbstractVisitor {
+
+ private final List<String> headings = new ArrayList<>();
+
+ @Nullable private String firstTopLevelHeading;
+
+ @Override
+ public void visit(Heading heading) {
+ String text = collectText(heading).trim();
+ if (!text.isEmpty()) {
+ headings.add(text);
+ if (firstTopLevelHeading == null &&
heading.getLevel() == 1) {
+ firstTopLevelHeading = text;
+ }
+ }
+ visitChildren(heading);
+ }
+
+ private static String collectText(Node node) {
+ StringBuilder builder = new StringBuilder();
+ appendText(node, builder);
+ return builder.toString();
+ }
+
+ private static void appendText(Node node, StringBuilder
builder) {
+ if (node instanceof Text text) {
+ builder.append(text.getLiteral());
+ } else if (node instanceof Code code) {
+ builder.append(code.getLiteral());
+ }
+ for (Node child = node.getFirstChild(); child != null;
child = child.getNext()) {
+ appendText(child, builder);
+ }
+ }
+ }
+}
diff --git
a/src/test/java/org/apache/solr/mcp/server/McpClientIntegrationTestBase.java
b/src/test/java/org/apache/solr/mcp/server/McpClientIntegrationTestBase.java
index d354082..3b9f230 100644
--- a/src/test/java/org/apache/solr/mcp/server/McpClientIntegrationTestBase.java
+++ b/src/test/java/org/apache/solr/mcp/server/McpClientIntegrationTestBase.java
@@ -104,6 +104,7 @@ public abstract class McpClientIntegrationTestBase {
assertTrue(toolNames.contains("create-collection"), "Should
have create-collection tool");
assertTrue(toolNames.contains("index-json-documents"), "Should
have index-json-documents tool");
+ assertTrue(toolNames.contains("index-markdown-documents"),
"Should have index-markdown-documents tool");
assertTrue(toolNames.contains("search"), "Should have search
tool");
assertTrue(toolNames.contains("list-collections"), "Should have
list-collections tool");
assertTrue(toolNames.contains("check-health"), "Should have
check-health tool");
@@ -135,6 +136,7 @@ public abstract class McpClientIntegrationTestBase {
assertHint(tools, "index-json-documents", false, true, true);
assertHint(tools, "index-csv-documents", false, true, true);
assertHint(tools, "index-xml-documents", false, true, true);
+ assertHint(tools, "index-markdown-documents", false, true,
true);
}
private static void assertReadOnly(Map<String, Tool> tools, String
name) {
@@ -398,6 +400,38 @@ public abstract class McpClientIntegrationTestBase {
assertEquals(1, getNumFound(r2), "Multi-valued 'genres' should
match on 'crime'");
}
+ // Self-contained markdown round-trip: independent of the other
order-18 test
+ // (the markdown document carries none of the fields its filter queries
touch).
+ @Test
+ @Order(18)
+ void indexMarkdownDocumentAndFindItById() throws Exception {
+ String markdown = """
+ ---
+ id: md-doc-1
+ author: Markdown Author
+ ---
+ # Markdown Indexing Guide
+
+ ## Installation
+
+ Index markdown documents through the MCP server.
+ """;
+
+ CallToolResult indexResult = mcpClient.callTool(new
CallToolRequest("index-markdown-documents",
+ Map.of("collection", COLLECTION, "markdown",
markdown)));
+
+ assertNotNull(indexResult);
+ assertNotError(indexResult);
+ assertTrue(extractText(indexResult).contains("Successfully
indexed 1"),
+ "Markdown indexing should report one indexed
document: " + extractText(indexResult));
+
+ CallToolResult searchResult = mcpClient
+ .callTool(new CallToolRequest("search",
Map.of("collection", COLLECTION, "query", "id:md-doc-1")));
+ Map<String, Object> response =
OBJECT_MAPPER.readValue(extractText(searchResult), new TypeReference<>() {
+ });
+ assertEquals(1, getNumFound(response), "Should find the
markdown document by its front matter id");
+ }
+
// ===== End-to-end shows workflow (orders 19–27) =====
// Exercises the canonical "set up a new collection with a defined
schema, index
// real data, then search and introspect" workflow purely through MCP
tool
diff --git
a/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceIntegrationTest.java
b/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceIntegrationTest.java
index e4f7516..ae29146 100644
---
a/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceIntegrationTest.java
+++
b/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceIntegrationTest.java
@@ -27,6 +27,7 @@ import org.apache.solr.mcp.server.TestcontainersConfiguration;
import org.apache.solr.mcp.server.indexing.documentcreator.CsvDocumentCreator;
import
org.apache.solr.mcp.server.indexing.documentcreator.IndexingDocumentCreator;
import org.apache.solr.mcp.server.indexing.documentcreator.JsonDocumentCreator;
+import
org.apache.solr.mcp.server.indexing.documentcreator.MarkdownDocumentCreator;
import org.apache.solr.mcp.server.indexing.documentcreator.XmlDocumentCreator;
import org.apache.solr.mcp.server.search.SearchResponse;
import org.apache.solr.mcp.server.search.SearchService;
@@ -72,9 +73,10 @@ class IndexingServiceIntegrationTest {
CsvDocumentCreator csvDocumentCreator = new
CsvDocumentCreator();
JsonDocumentCreator jsonDocumentCreator = new
JsonDocumentCreator(
new
com.fasterxml.jackson.databind.ObjectMapper());
+ MarkdownDocumentCreator markdownDocumentCreator = new
MarkdownDocumentCreator();
indexingDocumentCreator = new
IndexingDocumentCreator(xmlDocumentCreator, csvDocumentCreator,
- jsonDocumentCreator);
+ jsonDocumentCreator, markdownDocumentCreator);
indexingService = new IndexingService(solrClient,
indexingDocumentCreator);
searchService = new SearchService(solrClient);
diff --git
a/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java
b/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java
index 88c749e..b1491d1 100644
--- a/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java
+++ b/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java
@@ -354,6 +354,22 @@ class IndexingServiceTest {
assertTrue(body.contains("index-xml-documents"), "XML path
should reference index-xml-documents tool");
}
+ @Test
+ void indexDataPrompt_markdownPath_referencesIndexMarkdownDocuments() {
+ String body = indexingService.indexDataPrompt("library",
"markdown", null);
+
+ assertTrue(body.contains("index-markdown-documents"),
+ "Markdown path should reference
index-markdown-documents tool");
+ }
+
+ @Test
+ void indexDataPrompt_mdAliasResolvesToMarkdownTool() {
+ String body = indexingService.indexDataPrompt("library", "md",
null);
+
+ assertTrue(body.contains("index-markdown-documents"),
+ "'md' alias should reference
index-markdown-documents tool");
+ }
+
@Test
void indexDataPrompt_unknownFormat_throwsIllegalArgumentException() {
IllegalArgumentException ex =
assertThrows(IllegalArgumentException.class,
diff --git
a/src/test/java/org/apache/solr/mcp/server/indexing/MarkdownIndexingTest.java
b/src/test/java/org/apache/solr/mcp/server/indexing/MarkdownIndexingTest.java
new file mode 100644
index 0000000..2ec3a7c
--- /dev/null
+++
b/src/test/java/org/apache/solr/mcp/server/indexing/MarkdownIndexingTest.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.mcp.server.indexing;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.util.List;
+import org.apache.solr.common.SolrInputDocument;
+import
org.apache.solr.mcp.server.indexing.documentcreator.DocumentProcessingException;
+import
org.apache.solr.mcp.server.indexing.documentcreator.IndexingDocumentCreator;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.TestPropertySource;
+
+/**
+ * Test class for markdown indexing functionality in IndexingService.
+ *
+ * <p>
+ * This test verifies that the IndexingService can correctly parse markdown
+ * content — including YAML front matter, headings, and body text — and convert
+ * it into SolrInputDocument objects using the schema-less approach.
+ */
+@SpringBootTest
+@TestPropertySource(locations = "classpath:application.properties")
+class MarkdownIndexingTest {
+
+ @Autowired
+ private IndexingDocumentCreator indexingDocumentCreator;
+
+ @Test
+ void testCreateSchemalessDocumentsFromMarkdownWithFrontMatter() throws
Exception {
+ // Given
+ String markdown = """
+ ---
+ author: George R.R. Martin
+ genre: fantasy
+ published: 1996
+ ---
+ # A Game of Thrones
+
+ ## Synopsis
+
+ The first book in A Song of Ice and Fire.
+
+ ## Reception
+
+ Widely acclaimed.
+ """;
+
+ // When
+ List<SolrInputDocument> documents =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+
+ // Then
+ assertThat(documents).hasSize(1);
+
+ SolrInputDocument doc = documents.getFirst();
+ assertThat(doc.getFieldValue("author")).isEqualTo("George R.R.
Martin");
+ assertThat(doc.getFieldValue("genre")).isEqualTo("fantasy");
+ assertThat(doc.getFieldValue("published")).isEqualTo("1996");
+
+ // Title comes from the first level-1 heading (no front matter
title)
+ assertThat(doc.getFieldValue("title")).isEqualTo("A Game of
Thrones");
+
+ // All heading texts are collected into a multi-valued field
+ assertThat(doc.getFieldValues("headings")).containsExactly("A
Game of Thrones", "Synopsis", "Reception");
+
+ // Body text is searchable and excludes front matter
+ String content = (String) doc.getFieldValue("content");
+ assertThat(content).contains("The first book in A Song of Ice
and Fire.");
+ assertThat(content).contains("Widely acclaimed.");
+ assertThat(content).doesNotContain("George R.R. Martin");
+ }
+
+ @Test
+ void testFrontMatterTitleWinsOverFirstHeading() throws Exception {
+ // Given
+ String markdown = """
+ ---
+ title: Front Matter Title
+ ---
+ # Heading Title
+
+ Some body text.
+ """;
+
+ // When
+ List<SolrInputDocument> documents =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+
+ // Then
+ assertThat(documents).hasSize(1);
+
assertThat(documents.getFirst().getFieldValue("title")).isEqualTo("Front Matter
Title");
+ }
+
+ @Test
+ void testCreateSchemalessDocumentsFromMarkdownWithoutFrontMatter()
throws Exception {
+ // Given
+ String markdown = """
+ # Getting Started
+
+ Install the package and run the server.
+
+ ## Configuration
+
+ Set the `SOLR_URL` environment variable.
+ """;
+
+ // When
+ List<SolrInputDocument> documents =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+
+ // Then
+ assertThat(documents).hasSize(1);
+
+ SolrInputDocument doc = documents.getFirst();
+ assertThat(doc.getFieldValue("title")).isEqualTo("Getting
Started");
+
assertThat(doc.getFieldValues("headings")).containsExactly("Getting Started",
"Configuration");
+
+ String content = (String) doc.getFieldValue("content");
+ assertThat(content).contains("Install the package and run the
server.");
+ assertThat(content).contains("SOLR_URL");
+ }
+
+ @Test
+ void testPlainTextMarkdownWithoutHeadings() throws Exception {
+ // Given
+ String markdown = "Just a plain paragraph of text without any
structure.";
+
+ // When
+ List<SolrInputDocument> documents =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+
+ // Then
+ assertThat(documents).hasSize(1);
+
+ SolrInputDocument doc = documents.getFirst();
+ assertThat(doc.getFieldValue("title")).isNull();
+ assertThat(doc.getFieldValues("headings")).isNull();
+ assertThat(doc.getFieldValue("content")).isEqualTo("Just a
plain paragraph of text without any structure.");
+ }
+
+ @Test
+ void testFrontMatterFieldNamesAreSanitized() throws Exception {
+ // Given
+ String markdown = """
+ ---
+ Created-By: Jane Doe
+ last.updated: 2026-01-01
+ ---
+ # Doc
+
+ Body.
+ """;
+
+ // When
+ List<SolrInputDocument> documents =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+
+ // Then
+ assertThat(documents).hasSize(1);
+
+ SolrInputDocument doc = documents.getFirst();
+ assertThat(doc.getFieldValue("created_by")).isEqualTo("Jane
Doe");
+
assertThat(doc.getFieldValue("last_updated")).isEqualTo("2026-01-01");
+ }
+
+ @Test
+ void testFrontMatterListValuesBecomeMultiValuedFields() throws
Exception {
+ // Given
+ String markdown = """
+ ---
+ tags: [search, solr, mcp]
+ ---
+ # Tagged Document
+
+ Body.
+ """;
+
+ // When
+ List<SolrInputDocument> documents =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+
+ // Then
+ assertThat(documents).hasSize(1);
+
assertThat(documents.getFirst().getFieldValues("tags")).containsExactly("search",
"solr", "mcp");
+ }
+
+ @Test
+ void testFrontMatterBlockListValuesBecomeMultiValuedFields() throws
Exception {
+ // Given
+ String markdown = """
+ ---
+ tags:
+ - search
+ - solr
+ ---
+ # Tagged Document
+
+ Body.
+ """;
+
+ // When
+ List<SolrInputDocument> documents =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+
+ // Then
+ assertThat(documents).hasSize(1);
+
assertThat(documents.getFirst().getFieldValues("tags")).containsExactly("search",
"solr");
+ }
+
+ @Test
+ void testFrontMatterIdIsUsedAsDocumentId() throws Exception {
+ // Given
+ String markdown = """
+ ---
+ id: doc-42
+ ---
+ # Identified Document
+
+ Body.
+ """;
+
+ // When
+ List<SolrInputDocument> documents =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+
+ // Then
+ assertThat(documents).hasSize(1);
+
assertThat(documents.getFirst().getFieldValue("id")).isEqualTo("doc-42");
+ }
+
+ @Test
+ void testGeneratedIdIsStableForSameContent() throws Exception {
+ // Given
+ String markdown = "# Stable\n\nSame content, same id.";
+
+ // When
+ List<SolrInputDocument> first =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+ List<SolrInputDocument> second =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+ List<SolrInputDocument> other = indexingDocumentCreator
+ .createSchemalessDocumentsFromMarkdown("#
Different\n\nOther content.");
+
+ // Then: re-indexing the same markdown must overwrite the same
document
+ Object firstId = first.getFirst().getFieldValue("id");
+ assertThat(firstId).isNotNull();
+
assertThat(firstId).isEqualTo(second.getFirst().getFieldValue("id"));
+
assertThat(firstId).isNotEqualTo(other.getFirst().getFieldValue("id"));
+ }
+
+ @Test
+ void testEmptyMarkdownThrowsException() {
+ assertThatThrownBy(() ->
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(""))
+
.isInstanceOf(DocumentProcessingException.class).hasMessageContaining("cannot
be null or empty");
+ }
+
+ @Test
+ void testMarkdownFormattingIsStrippedFromContent() throws Exception {
+ // Given
+ String markdown = """
+ # Formatted
+
+ Some **bold** and *italic* text with a
[link](https://solr.apache.org) and `inline code`.
+ """;
+
+ // When
+ List<SolrInputDocument> documents =
indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+
+ // Then
+ assertThat(documents).hasSize(1);
+
+ String content = (String)
documents.getFirst().getFieldValue("content");
+ assertThat(content).contains("bold");
+ assertThat(content).contains("italic");
+ assertThat(content).contains("inline code");
+ assertThat(content).doesNotContain("**");
+ assertThat(content).doesNotContain("](");
+ }
+}
