This is an automated email from the ASF dual-hosted git repository. imbajin pushed a commit to branch goal-scan in repository https://gitbox.apache.org/repos/asf/hugegraph-ai.git
commit ed0f272c4df405bd5280c2e5e8e10c368d4116a4 Merge: 53f3ebac da710c8c Author: imbajin <[email protected]> AuthorDate: Tue Jun 2 20:53:46 2026 +0800 chore: merge latest goal-test into goal-scan - merge latest goal-test CI, fixture, and review updates - preserve goal-scan quality scan docs and runtime fixes - combine 401 auth handling with typed response errors - keep production smoke tests on namespaced fixtures .github/workflows/hugegraph-llm.yml | 40 +++++------ .github/workflows/hugegraph-python-client.yml | 28 ++++---- .workflow/quality-program/quality-state.json | 45 +++++++++++- .../reports/final-quality-report.md | 30 ++++++-- .../reports/production-change-ledger.md | 7 +- docs/quality/coverage-ratchet.md | 11 +++ hugegraph-llm/src/hugegraph_llm/api/rag_api.py | 2 + .../src/hugegraph_llm/config/models/base_config.py | 29 +++++++- .../config/models/base_prompt_config.py | 16 ++++- .../hugegraph_llm/demo/rag_demo/configs_block.py | 3 +- hugegraph-llm/src/tests/api/test_rag_api.py | 8 ++- hugegraph-llm/src/tests/config/test_config.py | 15 ++++ hugegraph-llm/src/tests/conftest.py | 58 +++++++++++++-- .../src/tests/integration/test_core_kg_smoke.py | 82 ++++++++++------------ .../integration/test_core_text2gremlin_smoke.py | 3 +- .../tests/integration/test_hugegraph_boundary.py | 60 +++------------- .../src/tests/integration/test_kg_construction.py | 3 +- .../operators/llm_op/test_gremlin_generate.py | 2 +- .../tests/operators/llm_op/test_keyword_extract.py | 2 +- .../llm_op/test_property_graph_extract.py | 2 +- .../src/pyhugegraph/utils/util.py | 3 + .../src/tests/api/test_response_validation.py | 16 ++++- 22 files changed, 309 insertions(+), 156 deletions(-) diff --cc hugegraph-llm/src/hugegraph_llm/api/rag_api.py index 9b9c7d35,4f5a80c3..d57f2812 --- a/hugegraph-llm/src/hugegraph_llm/api/rag_api.py +++ b/hugegraph-llm/src/hugegraph_llm/api/rag_api.py @@@ -168,8 -162,10 +168,10 @@@ def rag_http_api @router.post("/config/llm", status_code=status.HTTP_201_CREATED) def llm_config_api(req: LLMConfigRequest): llm_settings.chat_llm_type = req.llm_type + llm_settings.extract_llm_type = req.llm_type + llm_settings.text2gql_llm_type = req.llm_type - if req.llm_type == "openai": + if req.llm_type in ("openai", "litellm"): res = apply_llm_conf( req.api_key, req.api_base, diff --cc hugegraph-llm/src/tests/integration/test_kg_construction.py index cbca4102,379cb350..da793db2 --- a/hugegraph-llm/src/tests/integration/test_kg_construction.py +++ b/hugegraph-llm/src/tests/integration/test_kg_construction.py @@@ -15,89 -15,219 +15,90 @@@ # specific language governing permissions and limitations # under the License. -# pylint: disable=import-error,wrong-import-position,unused-argument - -import json -import os -import unittest -from unittest.mock import patch - import pytest - from fixtures.fake_llm import FakeLLM + -# 导入测试工具 -from src.tests.test_utils import ( - create_test_document, - should_skip_external, - with_mock_openai_client, -) - -pytestmark = [pytest.mark.external, pytest.mark.slow] - - -# Create mock classes to replace missing modules -class OpenAILLM: - """Mock OpenAILLM class""" - - def __init__(self, api_key=None, model=None): - self.api_key = api_key - self.model = model or "gpt-3.5-turbo" - - def generate(self, prompt): - # Return a mock response - return f"This is a mock response to '{prompt}'" - - -class KGConstructor: - """Mock KGConstructor class""" - - def __init__(self, llm, schema): - self.llm = llm - self.schema = schema - - def extract_entities(self, document): - # Mock entity extraction - if "张三" in document.content: - return [ - {"type": "Person", "name": "张三", "properties": {"occupation": "Software Engineer"}}, - { - "type": "Company", - "name": "ABC Company", - "properties": {"industry": "Technology", "location": "Beijing"}, - }, - ] - if "李四" in document.content: - return [ - {"type": "Person", "name": "李四", "properties": {"occupation": "Data Scientist"}}, - {"type": "Person", "name": "张三", "properties": {"occupation": "Software Engineer"}}, - ] - if "ABC Company" in document.content or "ABC公司" in document.content: - return [ - { - "type": "Company", - "name": "ABC Company", - "properties": {"industry": "Technology", "location": "Beijing"}, - } - ] - return [] - - def extract_relations(self, document): - # Mock relation extraction - if "张三" in document.content and ("ABC Company" in document.content or "ABC公司" in document.content): - return [ - { - "source": {"type": "Person", "name": "张三"}, - "relation": "works_for", - "target": {"type": "Company", "name": "ABC Company"}, - } - ] - if "李四" in document.content and "张三" in document.content: - return [ - { - "source": {"type": "Person", "name": "李四"}, - "relation": "colleague", - "target": {"type": "Person", "name": "张三"}, - } - ] - return [] - - def construct_from_documents(self, documents): - # Mock knowledge graph construction - entities = [] - relations = [] - - # Collect all entities and relations - for doc in documents: - entities.extend(self.extract_entities(doc)) - relations.extend(self.extract_relations(doc)) - - # Deduplicate entities - unique_entities = [] - entity_names = set() - for entity in entities: - if entity["name"] not in entity_names: - unique_entities.append(entity) - entity_names.add(entity["name"]) - - return {"entities": unique_entities, "relations": relations} - - -class TestKGConstruction(unittest.TestCase): - """Integration tests for knowledge graph construction""" - - def setUp(self): - """Setup work before testing""" - # Skip if external service tests should be skipped - if should_skip_external(): - self.skipTest("Skipping tests that require external services") - - # Load test schema - schema_path = os.path.join(os.path.dirname(__file__), "../data/kg/schema.json") - with open(schema_path, "r", encoding="utf-8") as f: - self.schema = json.load(f) - - # Create test documents - self.test_docs = [ - create_test_document("张三 is a software engineer working at ABC Company."), - create_test_document("李四 is 张三's colleague and works as a data scientist."), - create_test_document("ABC Company is a tech company headquartered in Beijing."), - ] - - # Create LLM model - self.llm = OpenAILLM() - - # Create knowledge graph constructor - self.kg_constructor = KGConstructor(llm=self.llm, schema=self.schema) - - @with_mock_openai_client - def test_entity_extraction(self, *args): - """Test entity extraction""" - # Extract entities from document - doc = self.test_docs[0] - entities = self.kg_constructor.extract_entities(doc) - - # Verify extracted entities - self.assertEqual(len(entities), 2) - self.assertEqual(entities[0]["name"], "张三") - self.assertEqual(entities[1]["name"], "ABC Company") - - @with_mock_openai_client - def test_relation_extraction(self, *args): - """Test relation extraction""" - # Extract relations from document - doc = self.test_docs[0] - relations = self.kg_constructor.extract_relations(doc) - - # Verify extracted relations - self.assertEqual(len(relations), 1) - self.assertEqual(relations[0]["source"]["name"], "张三") - self.assertEqual(relations[0]["relation"], "works_for") - self.assertEqual(relations[0]["target"]["name"], "ABC Company") - - @with_mock_openai_client - def test_kg_construction_end_to_end(self, *args): - """Test end-to-end knowledge graph construction process""" - # Mock entity and relation extraction - mock_entities = [ - {"type": "Person", "name": "张三", "properties": {"occupation": "Software Engineer"}}, - {"type": "Company", "name": "ABC Company", "properties": {"industry": "Technology"}}, - ] - - mock_relations = [ - { - "source": {"type": "Person", "name": "张三"}, - "relation": "works_for", - "target": {"type": "Company", "name": "ABC Company"}, - } ++from tests.fixtures.fake_llm import FakeLLM + +pytestmark = [pytest.mark.smoke, pytest.mark.integration] + + +PROPERTY_GRAPH_SCHEMA = { + "vertexlabels": [ + { + "id": 1, + "name": "person", + "properties": ["name", "occupation"], + "primary_keys": ["name"], + "nullable_keys": ["occupation"], + }, + { + "id": 2, + "name": "company", + "properties": ["name", "industry"], + "primary_keys": ["name"], + "nullable_keys": ["industry"], + }, + ], + "edgelabels": [ + { + "name": "works_for", + "source_label": "person", + "target_label": "company", + "properties": ["since"], + } + ], +} + + +def test_graph_extract_flow_builds_production_property_graph_pipeline(): + from hugegraph_llm.flows.graph_extract import GraphExtractFlow + + pipeline = GraphExtractFlow().build_flow( + schema=PROPERTY_GRAPH_SCHEMA, + texts=["Marko works for HugeGraph."], + example_prompt="extract property graph", + extract_type="property_graph", + language="en", + ) + prepared = pipeline.getGParamWithNoEmpty("wkflow_input") + dot = pipeline.dump() + + assert prepared.extract_type == "property_graph" + assert prepared.texts == ["Marko works for HugeGraph."] + assert 'label="schema_node"' in dot + assert 'label="chunk_split"' in dot + assert 'label="graph_extract"' in dot + + +def test_property_graph_extract_uses_production_operator_with_deterministic_llm(): + from hugegraph_llm.operators.llm_op.property_graph_extract import PropertyGraphExtract + + llm = FakeLLM( + [ + """{ + "vertices": [ + {"label": "person", "properties": {"name": "Marko", "occupation": "developer"}}, + {"label": "company", "properties": {"name": "HugeGraph", "industry": "graph"}} + ], + "edges": [ + { + "label": "works_for", + "properties": {"since": "2024"}, + "source": {"label": "person", "properties": {"name": "Marko"}}, + "target": {"label": "company", "properties": {"name": "HugeGraph"}} + } + ] + }""" ] - - # Mock KG constructor methods - with ( - patch.object(self.kg_constructor, "extract_entities", return_value=mock_entities), - patch.object(self.kg_constructor, "extract_relations", return_value=mock_relations), - ): - # Construct knowledge graph - use only one document to avoid duplicate relations from mocking - kg = self.kg_constructor.construct_from_documents([self.test_docs[0]]) - - # Verify knowledge graph - self.assertIsNotNone(kg) - self.assertEqual(len(kg["entities"]), 2) - self.assertEqual(len(kg["relations"]), 1) - - # Verify entities - entity_names = [e["name"] for e in kg["entities"]] - self.assertIn("张三", entity_names) - self.assertIn("ABC Company", entity_names) - - # Verify relations - relation = kg["relations"][0] - self.assertEqual(relation["source"]["name"], "张三") - self.assertEqual(relation["relation"], "works_for") - self.assertEqual(relation["target"]["name"], "ABC Company") - - def test_schema_validation(self): - """Test schema validation""" - # Verify schema structure - self.assertIn("vertices", self.schema) - self.assertIn("edges", self.schema) - - # Verify entity types - vertex_labels = [v["vertex_label"] for v in self.schema["vertices"]] - self.assertIn("person", vertex_labels) - - # Verify relation types - edge_labels = [e["edge_label"] for e in self.schema["edges"]] - self.assertIn("works_at", edge_labels) - - -if __name__ == "__main__": - unittest.main() + ) + context = PropertyGraphExtract(llm=llm, example_prompt=None).run( + { + "schema": PROPERTY_GRAPH_SCHEMA, + "chunks": ["Marko works for HugeGraph."], + } + ) + + assert [vertex["label"] for vertex in context["vertices"]] == ["person", "company"] + assert context["edges"][0]["label"] == "works_for" + assert context["edges"][0]["outV"] == "1:Marko" + assert context["edges"][0]["inV"] == "2:HugeGraph" diff --cc hugegraph-python-client/src/pyhugegraph/utils/util.py index 4d509cae,ffe0ff90..9cab72ff --- a/hugegraph-python-client/src/pyhugegraph/utils/util.py +++ b/hugegraph-python-client/src/pyhugegraph/utils/util.py @@@ -1,193 -1,143 +1,196 @@@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import json -import traceback - -import requests - -from pyhugegraph.utils.exceptions import ( - NotAuthorizedError, - NotFoundError, - ServiceUnavailableError, -) -from pyhugegraph.utils.log import log - - -def create_exception(response_content): - try: - data = json.loads(response_content) - if "ServiceUnavailableException" in data.get("exception", ""): - raise ServiceUnavailableError( - f'ServiceUnavailableException, "message": "{data["message"]}", "cause": "{data["cause"]}"' - ) - except (json.JSONDecodeError, KeyError) as e: - raise Exception(f"Error parsing response content: {response_content}") from e - raise Exception(response_content) - - -def check_if_authorized(response): - if response.status_code == 401: - raise NotAuthorizedError(f"Please check your username and password. {response.content!s}") - return True - - -def check_if_success(response, error=None): - if (not str(response.status_code).startswith("20")) and check_if_authorized(response): - if error is None: - error = NotFoundError(response.content) - - req = response.request - req_body = req.body if req.body else "Empty body" - response_body = response.text if response.text else "Empty body" - log.error( - "Error-Client: Request URL: %s, Request Body: %s, Response Body: %s", - req.url, - req_body, - response_body, - ) - raise error - return True - - -class ResponseValidation: - def __init__(self, content_type: str = "json", strict: bool = True) -> None: - super().__init__() - self._content_type = content_type - self._strict = strict - - def __call__(self, response: requests.Response, method: str, path: str): - """ - Validate the HTTP response according to the provided content type and strictness. - - :param response: HTTP response object - :param method: HTTP method used (e.g., 'GET', 'POST') - :param path: URL path of the request - :return: Parsed response content or empty dict if none applicable - """ - result = {} - - try: - response.raise_for_status() - if response.status_code == 204: - log.debug("No content returned (204) for %s: %s", method, path) - else: - if self._content_type == "raw": - result = response - elif self._content_type == "json": - result = response.json() - elif self._content_type == "text": - result = response.text - else: - raise ValueError(f"Unknown content type: {self._content_type}") - - except requests.exceptions.HTTPError as e: - if not self._strict and response.status_code == 404: - log.info("Resource %s not found (404)", path) - else: - if response.status_code == 401: - check_if_authorized(response) - - try: - body = response.json() - if isinstance(body, dict): - status = body.get("status") - status_message = status.get("message") if isinstance(status, dict) else None - details = ( - body.get("message") - or body.get("exception") - or status_message - or response.text - or "unknown error" - ) - else: - details = response.text or "unknown error" - except (ValueError, KeyError, AttributeError, TypeError): - details = response.text or "unknown error" - - req_body = response.request.body if response.request.body else "Empty body" - req_body = req_body.encode("utf-8").decode("unicode_escape") - log.error( - "%s: %s\n[Body]: %s\n[Server Exception]: %s", - method, - str(e).encode("utf-8").decode("unicode_escape"), - req_body, - details, - ) - - if response.status_code == 404: - raise NotFoundError(response.content) from e - raise Exception(f"Server Exception: {details}") from e - - except Exception: # pylint: disable=broad-exception-caught - log.error("Unhandled exception occurred: %s", traceback.format_exc()) - - return result - - def __repr__(self) -> str: - return f"ResponseValidation(content_type={self._content_type}, strict={self._strict})" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import json +import re +import traceback + +import requests + +from pyhugegraph.utils.exceptions import ( + NotAuthorizedError, + NotFoundError, + ResponseParseError, + ServerError, + ServiceUnavailableError, +) +from pyhugegraph.utils.log import log + +REDACTED_VALUE = "***REDACTED***" +SENSITIVE_KEY_PARTS = ( + "api_key", + "authorization", + "password", + "passwd", + "pwd", + "secret", + "token", +) + + +def _is_sensitive_key(key) -> bool: + key_lower = str(key).lower() + return any(part in key_lower for part in SENSITIVE_KEY_PARTS) + + +def redact_sensitive_data(value): + if isinstance(value, dict): + return { + key: REDACTED_VALUE if _is_sensitive_key(key) else redact_sensitive_data(item) + for key, item in value.items() + } + if isinstance(value, list): + return [redact_sensitive_data(item) for item in value] + if isinstance(value, tuple): + return tuple(redact_sensitive_data(item) for item in value) + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + if isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + redacted = re.sub( + r'(?i)("?[a-z0-9_-]*(?:api_key|authorization|password|passwd|pwd|secret|token)[a-z0-9_-]*"?\s*[:=]\s*)"[^"]*"', + rf"\1\"{REDACTED_VALUE}\"", + value, + ) + return re.sub( + r"(?i)(api_key|authorization|password|passwd|pwd|secret|token)=([^&\s]+)", + rf"\1={REDACTED_VALUE}", + redacted, + ) + return json.dumps(redact_sensitive_data(parsed), ensure_ascii=False) + return value + + +def create_exception(response_content): + try: + data = json.loads(response_content) + if "ServiceUnavailableException" in data.get("exception", ""): + raise ServiceUnavailableError( + f'ServiceUnavailableException, "message": "{data["message"]}", "cause": "{data["cause"]}"' + ) + except (json.JSONDecodeError, KeyError) as e: + raise Exception(f"Error parsing response content: {response_content}") from e + raise Exception(response_content) + + +def check_if_authorized(response): + if response.status_code == 401: + raise NotAuthorizedError(f"Please check your username and password. {response.content!s}") + return True + + +def check_if_success(response, error=None): + if (not str(response.status_code).startswith("20")) and check_if_authorized(response): + if error is None: + error = NotFoundError(response.content) + + req = response.request + req_body = redact_sensitive_data(req.body) if req.body else "Empty body" + response_body = response.text if response.text else "Empty body" + log.error( + "Error-Client: Request URL: %s, Request Body: %s, Response Body: %s", + req.url, + req_body, + response_body, + ) + raise error + return True + + +class ResponseValidation: + def __init__(self, content_type: str = "json", strict: bool = True) -> None: + super().__init__() + self._content_type = content_type + self._strict = strict + + def __call__(self, response: requests.Response, method: str, path: str): + """ + Validate the HTTP response according to the provided content type and strictness. + + :param response: HTTP response object + :param method: HTTP method used (e.g., 'GET', 'POST') + :param path: URL path of the request + :return: Parsed response content or empty dict if none applicable + """ + result = {} + + try: + response.raise_for_status() + if response.status_code == 204: + log.debug("No content returned (204) for %s: %s", method, path) + else: + if self._content_type == "raw": + result = response + elif self._content_type == "json": + result = response.json() + elif self._content_type == "text": + result = response.text + else: + raise ValueError(f"Unknown content type: {self._content_type}") + + except requests.exceptions.HTTPError as e: + if not self._strict and response.status_code == 404: + log.info("Resource %s not found (404)", path) + else: ++ if response.status_code == 401: ++ check_if_authorized(response) ++ + try: + body = response.json() + if isinstance(body, dict): + status = body.get("status") + status_message = status.get("message") if isinstance(status, dict) else None + details = ( + body.get("message") + or body.get("exception") + or status_message + or response.text + or "unknown error" + ) + else: + details = response.text or "unknown error" + except (ValueError, KeyError, AttributeError, TypeError): + details = response.text or "unknown error" + + req_body = redact_sensitive_data(response.request.body) if response.request.body else "Empty body" + if isinstance(req_body, str): + req_body = req_body.encode("utf-8").decode("unicode_escape") + log.error( + "%s: %s\n[Body]: %s\n[Server Exception]: %s", + method, + str(e).encode("utf-8").decode("unicode_escape"), + req_body, + details, + ) + + if response.status_code == 404: + raise NotFoundError(response.content) from e + if response.status_code >= 400: + raise ServerError(f"Server Exception: {details}") from e + raise e + + except Exception as e: + log.error("Unhandled exception occurred: %s", traceback.format_exc()) + raise ResponseParseError(f"Failed to parse {self._content_type} response for {method} {path}") from e + + return result + + def __repr__(self) -> str: + return f"ResponseValidation(content_type={self._content_type}, strict={self._strict})" diff --cc hugegraph-python-client/src/tests/api/test_response_validation.py index 0653db55,77dc3c9a..72b76a2f --- a/hugegraph-python-client/src/tests/api/test_response_validation.py +++ b/hugegraph-python-client/src/tests/api/test_response_validation.py @@@ -20,7 -20,7 +20,7 @@@ from unittest.mock import Moc import pytest import requests - from pyhugegraph.utils.exceptions import ResponseParseError, ServerError -from pyhugegraph.utils.exceptions import NotAuthorizedError ++from pyhugegraph.utils.exceptions import NotAuthorizedError, ResponseParseError, ServerError from pyhugegraph.utils.util import ResponseValidation pytestmark = pytest.mark.contract @@@ -78,33 -78,20 +78,47 @@@ class TestResponseValidation(unittest.T with self.assertRaisesRegex(Exception, "Server Exception: not json"): validator(response, "POST", "/gremlin") + def test_malformed_success_json_raises_parse_error(self): + response = Mock(spec=requests.Response) + response.status_code = 200 + response.text = "not json" + response.content = b"not json" + response.json.side_effect = ValueError("not json") + response.raise_for_status.return_value = None + validator = ResponseValidation() + + with pytest.raises(ResponseParseError, match="Failed to parse json response"): + validator(response, "GET", "/graphs/hugegraph") + + def test_error_log_redacts_sensitive_request_body(self): + response = self._mock_error_response( + {"message": "bad request"}, + '{"message":"bad request"}', + ) + response.request.body = '{"user_name":"marko","user_password":"super-secret"}' + validator = ResponseValidation() + + with pytest.raises(ServerError), unittest.mock.patch("pyhugegraph.utils.util.log.error") as log_error: + validator(response, "POST", "/auth/users") + + logged_args = str(log_error.call_args) + assert "super-secret" not in logged_args + assert "***REDACTED***" in logged_args + + def test_unauthorized_error_preserves_not_authorized_type(self): + response = Mock(spec=requests.Response) + response.status_code = 401 + response.text = '{"exception":"NotAuthorizedException","message":"Authentication failed"}' + response.content = response.text.encode("utf-8") + response.request = Mock(body="Empty body", url="http://127.0.0.1:8080/graphs";) + response.raise_for_status.side_effect = requests.exceptions.HTTPError("401 Client Error") + validator = ResponseValidation() + + with pytest.raises(NotAuthorizedError) as exc_info: + validator(response, method="GET", path="/graphs") + + assert "Please check your username and password" in str(exc_info.value) + if __name__ == "__main__": unittest.main()
