This is an automated email from the ASF dual-hosted git repository.
imbajin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hugegraph-ai.git
The following commit(s) were added to refs/heads/main by this push:
new 733f389d feat: support PDF uploads for RAG documents (#356)
733f389d is described below
commit 733f389df6d3b2d065b873765b986b1719e79e14
Author: Nannan Wang <[email protected]>
AuthorDate: Mon Jun 8 12:29:19 2026 +0800
feat: support PDF uploads for RAG documents (#356)
### Summary
Support text-based PDF uploads in the RAG document upload path.
### Changes
* Add PDF text extraction support in `read_documents()`
* Use `pypdf` to extract text from PDF files page by page
* Handle encrypted, unreadable, and scanned-image-only PDFs with clear
Gradio errors
* Keep existing TXT and DOCX behavior unchanged
* Update the demo upload copy to mention TXT, DOCX, and PDF
### Test
* `python -m py_compile
hugegraph-llm/src/hugegraph_llm/utils/vector_index_utils.py`
Closes #345
---------
Co-authored-by: nannan-2026 <[email protected]>
Co-authored-by: imbajin <[email protected]>
---
hugegraph-llm/pyproject.toml | 1 +
.../demo/rag_demo/vector_graph_block.py | 2 +-
.../src/hugegraph_llm/utils/vector_index_utils.py | 41 ++-
.../src/tests/document/test_vector_index_utils.py | 288 +++++++++++++++++++++
pyproject.toml | 1 +
5 files changed, 326 insertions(+), 7 deletions(-)
diff --git a/hugegraph-llm/pyproject.toml b/hugegraph-llm/pyproject.toml
index 5b2c9f5d..cd2452d9 100644
--- a/hugegraph-llm/pyproject.toml
+++ b/hugegraph-llm/pyproject.toml
@@ -53,6 +53,7 @@ dependencies = [
"gradio",
"jieba",
"python-docx",
+ "pypdf",
"langchain-text-splitters",
"faiss-cpu",
"python-dotenv",
diff --git
a/hugegraph-llm/src/hugegraph_llm/demo/rag_demo/vector_graph_block.py
b/hugegraph-llm/src/hugegraph_llm/demo/rag_demo/vector_graph_block.py
index 5dbd87c4..6816d9f4 100644
--- a/hugegraph-llm/src/hugegraph_llm/demo/rag_demo/vector_graph_block.py
+++ b/hugegraph-llm/src/hugegraph_llm/demo/rag_demo/vector_graph_block.py
@@ -216,7 +216,7 @@ def create_vector_graph_block():
"""## Build Vector/Graph Index & Extract Knowledge Graph
- Docs:
- text: Build rag index from plain text
- - file: Upload file(s) which should be <u>TXT</u> or <u>.docx</u>
(Multiple files can be selected together)
+ - file: Upload file(s) which should be <u>TXT</u>, <u>DOCX</u>, or
<u>PDF</u> (Multiple files can be selected together)
- [Schema](https://hugegraph.apache.org/docs/clients/restful-api/schema/):
(Accept **2 types**)
- User-defined Schema (JSON format, follow the
[template](https://github.com/apache/hugegraph-ai/blob/aff3bbe25fa91c3414947a196131be812c20ef11/hugegraph-llm/src/hugegraph_llm/config/config_data.py#L125)
to modify it)
diff --git a/hugegraph-llm/src/hugegraph_llm/utils/vector_index_utils.py
b/hugegraph-llm/src/hugegraph_llm/utils/vector_index_utils.py
index bb9536d2..7475cdd0 100644
--- a/hugegraph-llm/src/hugegraph_llm/utils/vector_index_utils.py
+++ b/hugegraph-llm/src/hugegraph_llm/utils/vector_index_utils.py
@@ -16,10 +16,12 @@
# under the License.
import json
+from pathlib import Path
from typing import Type
import docx
import gradio as gr
+from pypdf import PdfReader
from hugegraph_llm.config import huge_settings, index_settings
from hugegraph_llm.flows.scheduler import SchedulerSingleton
@@ -28,6 +30,33 @@ from hugegraph_llm.indices.vector_index.faiss_vector_store
import FaissVectorInd
from hugegraph_llm.models.embeddings.init_embedding import Embeddings
+def read_pdf_text(full_path: str) -> str:
+ try:
+ with open(full_path, "rb") as pdf_file:
+ reader = PdfReader(pdf_file)
+
+ if reader.is_encrypted:
+ raise gr.Error("Encrypted PDF files are not supported. Please
upload an unencrypted PDF.")
+
+ page_texts = []
+ for page in reader.pages:
+ page_text = page.extract_text() or ""
+ if page_text.strip():
+ page_texts.append(page_text)
+
+ text = "\n".join(page_texts).strip()
+ if not text:
+ raise gr.Error(
+ "No extractable text was found in this PDF. Scanned-image
PDFs are not supported without OCR."
+ )
+
+ return text
+ except gr.Error:
+ raise
+ except Exception as exc:
+ raise gr.Error(f"Failed to read PDF file: {exc}") from exc
+
+
def read_documents(input_file, input_text):
if input_text:
texts = [input_text]
@@ -35,21 +64,21 @@ def read_documents(input_file, input_text):
texts = []
for file in input_file:
full_path = file.name
- if full_path.endswith(".txt"):
+ suffix = Path(full_path).suffix.lower()
+ if suffix == ".txt":
with open(full_path, "r", encoding="utf-8") as f:
texts.append(f.read())
- elif full_path.endswith(".docx"):
+ elif suffix == ".docx":
text = ""
doc = docx.Document(full_path)
for para in doc.paragraphs:
text += para.text
text += "\n"
texts.append(text)
- elif full_path.endswith(".pdf"):
- # TODO: support PDF file
- raise gr.Error("PDF will be supported later! Try to upload
text/docx now")
+ elif suffix == ".pdf":
+ texts.append(read_pdf_text(full_path))
else:
- raise gr.Error("Please input txt or docx file.")
+ raise gr.Error("Please input txt, docx, or pdf file.")
else:
raise gr.Error("Please input text or upload file.")
return texts
diff --git a/hugegraph-llm/src/tests/document/test_vector_index_utils.py
b/hugegraph-llm/src/tests/document/test_vector_index_utils.py
new file mode 100644
index 00000000..33213cdc
--- /dev/null
+++ b/hugegraph-llm/src/tests/document/test_vector_index_utils.py
@@ -0,0 +1,288 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from types import SimpleNamespace
+
+import gradio as gr
+import pytest
+from docx import Document
+from pypdf import PdfWriter
+
+from hugegraph_llm.utils import graph_index_utils, vector_index_utils
+from hugegraph_llm.utils.vector_index_utils import read_documents
+
+
+def _build_pdf(content_stream: bytes) -> bytes:
+ objects = [
+ b"<< /Type /Catalog /Pages 2 0 R >>",
+ b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+ (
+ b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
+ b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>"
+ ),
+ b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+ (b"<< /Length " + str(len(content_stream)).encode() + b" >>\nstream\n"
+ content_stream + b"\nendstream"),
+ ]
+
+ pdf = b"%PDF-1.4\n"
+ offsets = []
+ for index, obj in enumerate(objects, start=1):
+ offsets.append(len(pdf))
+ pdf += f"{index} 0 obj\n".encode()
+ pdf += obj + b"\nendobj\n"
+
+ xref_offset = len(pdf)
+ pdf += f"xref\n0 {len(objects) + 1}\n".encode()
+ pdf += b"0000000000 65535 f \n"
+ for offset in offsets:
+ pdf += f"{offset:010d} 00000 n \n".encode()
+
+ pdf += (f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R
>>\nstartxref\n{xref_offset}\n%%EOF\n").encode()
+ return pdf
+
+
+def test_read_documents_reads_txt_file(tmp_path):
+ txt_path = tmp_path / "sample.txt"
+ txt_path.write_text("hello hugegraph", encoding="utf-8")
+
+ result = read_documents([SimpleNamespace(name=str(txt_path))], "")
+
+ assert result == ["hello hugegraph"]
+
+
+def _escape_pdf_text(text):
+ return text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
+
+
+def _build_multi_page_pdf(page_texts):
+ page_count = len(page_texts)
+ page_object_start = 4
+ content_object_start = page_object_start + page_count
+ kids = " ".join(f"{page_object_start + index} 0 R" for index in
range(page_count))
+
+ objects = [
+ b"<< /Type /Catalog /Pages 2 0 R >>",
+ f"<< /Type /Pages /Kids [{kids}] /Count {page_count} >>".encode(),
+ b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+ ]
+
+ for index in range(page_count):
+ content_ref = content_object_start + index
+ objects.append(
+ (
+ "<< /Type /Page /Parent 2 0 R "
+ "/Resources << /Font << /F1 3 0 R >> >> "
+ f"/Contents {content_ref} 0 R "
+ "/MediaBox [0 0 612 792] >>"
+ ).encode()
+ )
+
+ for page_text in page_texts:
+ content_stream = (f"BT /F1 24 Tf 72 720 Td
({_escape_pdf_text(page_text)}) Tj ET").encode()
+ objects.append(
+ b"<< /Length " + str(len(content_stream)).encode() + b"
>>\nstream\n" + content_stream + b"\nendstream"
+ )
+
+ pdf = b"%PDF-1.4\n"
+ offsets = []
+ for object_id, pdf_object in enumerate(objects, start=1):
+ offsets.append(len(pdf))
+ pdf += f"{object_id} 0 obj\n".encode()
+ pdf += pdf_object + b"\nendobj\n"
+
+ xref_offset = len(pdf)
+ pdf += f"xref\n0 {len(objects) + 1}\n".encode()
+ pdf += b"0000000000 65535 f \n"
+ for offset in offsets:
+ pdf += f"{offset:010d} 00000 n \n".encode()
+
+ pdf += (f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R
>>\nstartxref\n{xref_offset}\n%%EOF\n").encode()
+ return pdf
+
+
+def test_read_documents_reads_pdf_file(tmp_path):
+ pdf_path = tmp_path / "sample.pdf"
+ pdf_path.write_bytes(_build_pdf(b"BT /F1 24 Tf 100 700 Td (Hello HugeGraph
PDF) Tj ET"))
+
+ result = read_documents([SimpleNamespace(name=str(pdf_path))], "")
+
+ assert "Hello HugeGraph PDF" in result[0]
+
+
+def test_read_documents_reads_mixed_case_pdf_file(tmp_path):
+ pdf_path = tmp_path / "sample.PDF"
+ pdf_path.write_bytes(_build_pdf(b"BT /F1 24 Tf 100 700 Td (Mixed Case PDF)
Tj ET"))
+
+ result = read_documents([SimpleNamespace(name=str(pdf_path))], "")
+
+ assert "Mixed Case PDF" in result[0]
+
+
+def test_read_documents_rejects_pdf_without_extractable_text(tmp_path):
+ pdf_path = tmp_path / "empty.pdf"
+ pdf_path.write_bytes(_build_pdf(b""))
+
+ with pytest.raises(gr.Error, match="No extractable text"):
+ read_documents([SimpleNamespace(name=str(pdf_path))], "")
+
+
+class BrokenPdfReader:
+ def __init__(self, _):
+ raise ValueError("broken pdf")
+
+
+def test_read_documents_reads_docx_file(tmp_path):
+ docx_path = tmp_path / "sample.docx"
+ document = Document()
+ document.add_paragraph("hello docx")
+ document.save(docx_path)
+
+ result = read_documents([SimpleNamespace(name=str(docx_path))], "")
+
+ assert "hello docx" in result[0]
+
+
+def test_read_documents_returns_clear_error_for_unreadable_pdf(monkeypatch,
tmp_path):
+ pdf_path = tmp_path / "broken.pdf"
+ pdf_path.write_bytes(b"not a valid pdf")
+
+ monkeypatch.setattr(vector_index_utils, "PdfReader", BrokenPdfReader)
+
+ with pytest.raises(gr.Error, match="Failed to read PDF file"):
+ read_documents([SimpleNamespace(name=str(pdf_path))], "")
+
+
+class EncryptedPdfReader:
+ is_encrypted = True
+
+ def __init__(self, _):
+ self.pages = []
+
+
+def test_read_documents_returns_clear_error_for_encrypted_pdf(monkeypatch,
tmp_path):
+ pdf_path = tmp_path / "encrypted.pdf"
+ pdf_path.write_bytes(b"%PDF-1.4\n")
+
+ monkeypatch.setattr(vector_index_utils, "PdfReader", EncryptedPdfReader)
+
+ with pytest.raises(gr.Error, match="Encrypted PDF files are not
supported"):
+ read_documents([SimpleNamespace(name=str(pdf_path))], "")
+
+
+def test_read_documents_rejects_unsupported_file_type(tmp_path):
+ markdown_path = tmp_path / "sample.md"
+ markdown_path.write_text("hello markdown", encoding="utf-8")
+
+ with pytest.raises(gr.Error, match="Please input txt, docx, or pdf file"):
+ read_documents([SimpleNamespace(name=str(markdown_path))], "")
+
+
+class DummyScheduler:
+ def __init__(self):
+ self.calls = []
+
+ def schedule_flow(self, *args):
+ self.calls.append(args)
+ return "scheduled"
+
+
+def test_build_vector_index_accepts_pdf_upload_and_forwards_text(monkeypatch,
tmp_path):
+ pdf_path = tmp_path / "entrypoint.pdf"
+ pdf_path.write_bytes(_build_pdf(b"BT /F1 24 Tf 100 700 Td (Build Vector
Entrypoint PDF) Tj ET"))
+ scheduler = DummyScheduler()
+
+ monkeypatch.setattr(
+ vector_index_utils.SchedulerSingleton,
+ "get_instance",
+ lambda: scheduler,
+ )
+
+ result = vector_index_utils.build_vector_index(
+ [SimpleNamespace(name=str(pdf_path))],
+ "",
+ )
+
+ assert result == "scheduled"
+ assert len(scheduler.calls) == 1
+
+ flow_name, texts = scheduler.calls[0]
+ assert flow_name == "build_vector_index"
+ assert "Build Vector Entrypoint PDF" in texts[0]
+
+
+def test_extract_graph_accepts_pdf_upload_and_forwards_text(monkeypatch,
tmp_path):
+ pdf_path = tmp_path / "graph_entrypoint.pdf"
+ pdf_path.write_bytes(_build_pdf(b"BT /F1 24 Tf 100 700 Td (Extract Graph
Entrypoint PDF) Tj ET"))
+ schema = '{"vertices": [], "edges": []}'
+ example_prompt = "Extract graph data."
+ scheduler = DummyScheduler()
+
+ monkeypatch.setattr(
+ graph_index_utils.SchedulerSingleton,
+ "get_instance",
+ lambda: scheduler,
+ )
+
+ result = graph_index_utils.extract_graph(
+ [SimpleNamespace(name=str(pdf_path))],
+ "",
+ schema,
+ example_prompt,
+ )
+
+ assert result == "scheduled"
+ assert len(scheduler.calls) == 1
+
+ flow_name, forwarded_schema, texts, forwarded_prompt, graph_mode =
scheduler.calls[0]
+ assert flow_name == graph_index_utils.FlowName.GRAPH_EXTRACT
+ assert forwarded_schema == schema
+ assert "Extract Graph Entrypoint PDF" in texts[0]
+ assert forwarded_prompt == example_prompt
+ assert graph_mode == "property_graph"
+
+
+def test_read_documents_rejects_encrypted_pdf(tmp_path):
+ pdf_path = tmp_path / "encrypted.pdf"
+
+ writer = PdfWriter()
+ writer.add_blank_page(width=72, height=72)
+ writer.encrypt("secret")
+ with pdf_path.open("wb") as pdf_file:
+ writer.write(pdf_file)
+
+ with pytest.raises(gr.Error) as exc_info:
+
vector_index_utils.read_documents([SimpleNamespace(name=str(pdf_path))], "")
+
+ assert "PDF" in str(exc_info.value)
+
+
+def test_read_documents_preserves_multi_page_pdf_order(tmp_path):
+ pdf_path = tmp_path / "multi_page.pdf"
+ pdf_path.write_bytes(
+ _build_multi_page_pdf(
+ [
+ "First page PDF text",
+ "Second page PDF text",
+ "Third page PDF text",
+ ]
+ )
+ )
+
+ result =
vector_index_utils.read_documents([SimpleNamespace(name=str(pdf_path))], "")
+
+ assert len(result) == 1
+ extracted_text = result[0]
+ assert extracted_text.index("First page PDF text") <
extracted_text.index("Second page PDF text")
+ assert extracted_text.index("Second page PDF text") <
extracted_text.index("Third page PDF text")
diff --git a/pyproject.toml b/pyproject.toml
index 2687ce93..c2feda54 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -119,6 +119,7 @@ constraint-dependencies = [
"gradio~=5.20.0",
"jieba~=0.42.1",
"python-docx~=1.1.2",
+ "pypdf~=6.12.0",
"langchain-text-splitters~=0.2.2",
"faiss-cpu~=1.8.0",
"python-dotenv~=1.0.1",
