This is an automated email from the ASF dual-hosted git repository.
jin pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-ai.git
from 2eb7834b feat(llm): update keyword extraction method (BREAKING CHANGE)
(#282)
new 0bdbf899 refactor: refactor scheduler to support dynamic workflow
scheduling and pipeline pooling (#48)
new b90925ad refactor: refactor hugegraph-ai to integrate with CGraph &
port some usecases in web demo (#49)
new d36d41d1 refactor: text2germlin with PCGraph framework (#50)
new 6ad8fd9b refactor(RAG workflow): modularize flows, add streaming, and
improve node initialization (#51)
new edae3895 refactor: port batch build gremlin examples & delete some doc
related to Pipeline(old design) & refactor some operator's design and
implementation & code format
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 36 --
hugegraph-llm/README.md | 155 +++--
hugegraph-llm/pyproject.toml | 6 +-
hugegraph-llm/src/hugegraph_llm/api/admin_api.py | 8 +-
.../hugegraph_llm/api/exceptions/rag_exceptions.py | 4 +-
.../src/hugegraph_llm/api/models/rag_requests.py | 106 +++-
hugegraph-llm/src/hugegraph_llm/api/rag_api.py | 77 ++-
.../src/hugegraph_llm/config/admin_config.py | 2 +
hugegraph-llm/src/hugegraph_llm/config/generate.py | 4 +-
.../src/hugegraph_llm/config/hugegraph_config.py | 1 +
.../src/hugegraph_llm/config/llm_config.py | 21 +-
.../src/hugegraph_llm/config/models/base_config.py | 22 +-
.../config/models/base_prompt_config.py | 24 +-
.../src/hugegraph_llm/config/prompt_config.py | 1 +
.../src/hugegraph_llm/demo/rag_demo/admin_block.py | 39 +-
.../src/hugegraph_llm/demo/rag_demo/app.py | 4 +-
.../hugegraph_llm/demo/rag_demo/configs_block.py | 91 +--
.../src/hugegraph_llm/demo/rag_demo/other_block.py | 13 +-
.../src/hugegraph_llm/demo/rag_demo/rag_block.py | 243 +++++---
.../demo/rag_demo/text2gremlin_block.py | 249 ++++----
.../demo/rag_demo/vector_graph_block.py | 38 +-
.../src/hugegraph_llm/document/chunk_split.py | 14 +-
.../{enums/id_strategy.py => flows/__init__.py} | 21 +-
.../src/hugegraph_llm/flows/build_example_index.py | 62 ++
.../src/hugegraph_llm/flows/build_schema.py | 75 +++
.../src/hugegraph_llm/flows/build_vector_index.py | 55 ++
hugegraph-llm/src/hugegraph_llm/flows/common.py | 64 ++
.../hugegraph_llm/flows/get_graph_index_info.py | 70 +++
.../src/hugegraph_llm/flows/graph_extract.py | 92 +++
.../src/hugegraph_llm/flows/import_graph_data.py | 67 +++
.../src/hugegraph_llm/flows/prompt_generate.py | 64 ++
.../src/hugegraph_llm/flows/rag_flow_graph_only.py | 173 ++++++
.../hugegraph_llm/flows/rag_flow_graph_vector.py | 147 +++++
.../src/hugegraph_llm/flows/rag_flow_raw.py | 88 +++
.../hugegraph_llm/flows/rag_flow_vector_only.py | 112 ++++
hugegraph-llm/src/hugegraph_llm/flows/scheduler.py | 189 ++++++
.../src/hugegraph_llm/flows/text2gremlin.py | 116 ++++
.../hugegraph_llm/flows/update_vid_embeddings.py | 49 ++
.../src/hugegraph_llm/indices/graph_index.py | 17 +-
.../src/hugegraph_llm/indices/vector_index.py | 33 +-
.../src/hugegraph_llm/middleware/middleware.py | 5 +-
.../src/hugegraph_llm/models/embeddings/base.py | 37 +-
.../models/embeddings/init_embedding.py | 36 +-
.../src/hugegraph_llm/models/embeddings/openai.py | 25 +-
.../src/hugegraph_llm/models/llms/base.py | 34 +-
.../src/hugegraph_llm/models/llms/init_llm.py | 80 ++-
.../src/hugegraph_llm/models/llms/litellm.py | 10 +-
.../src/hugegraph_llm/models/llms/ollama.py | 30 +-
.../src/hugegraph_llm/models/llms/openai.py | 4 +-
.../src/hugegraph_llm/models/rerankers/cohere.py | 9 +-
.../models/rerankers/init_reranker.py | 4 +-
.../hugegraph_llm/models/rerankers/siliconflow.py | 9 +-
hugegraph-llm/src/hugegraph_llm/nodes/base_node.py | 101 ++++
.../nodes/common_node/merge_rerank_node.py | 85 +++
.../nodes/document_node/chunk_split.py | 43 ++
.../nodes/hugegraph_node/commit_to_hugegraph.py | 34 ++
.../nodes/hugegraph_node/fetch_graph_data.py | 35 ++
.../hugegraph_node/graph_query_node.py} | 597 ++++++++++---------
.../nodes/hugegraph_node/gremlin_execute.py | 68 +++
.../hugegraph_llm/nodes/hugegraph_node/schema.py | 69 +++
.../index_node/build_gremlin_example_index.py | 43 ++
.../nodes/index_node/build_semantic_index.py | 33 ++
.../nodes/index_node/build_vector_index.py | 33 ++
.../index_node/gremlin_example_index_query.py | 54 ++
.../nodes/index_node/semantic_id_query_node.py | 93 +++
.../nodes/index_node/vector_query_node.py | 69 +++
.../nodes/llm_node/answer_synthesize_node.py | 86 +++
.../hugegraph_llm/nodes/llm_node/extract_info.py | 53 ++
.../nodes/llm_node/keyword_extract_node.py | 67 +++
.../nodes/llm_node/prompt_generate.py | 59 ++
.../hugegraph_llm/nodes/llm_node/schema_build.py | 91 +++
.../hugegraph_llm/nodes/llm_node/text2gremlin.py | 72 +++
hugegraph-llm/src/hugegraph_llm/nodes/util.py | 38 ++
.../operators/common_op/check_schema.py | 58 +-
.../operators/common_op/merge_dedup_rerank.py | 15 +-
.../operators/document_op/chunk_split.py | 1 +
.../operators/document_op/word_extract.py | 3 +-
.../operators/gremlin_generate_task.py | 81 ---
.../operators/hugegraph_op/commit_to_hugegraph.py | 126 +++-
.../operators/hugegraph_op/schema_manager.py | 11 +-
.../index_op/build_gremlin_example_index.py | 14 +-
.../operators/index_op/build_semantic_index.py | 24 +-
.../operators/index_op/build_vector_index.py | 13 +-
.../index_op/gremlin_example_index_query.py | 29 +-
.../operators/index_op/semantic_id_query.py | 33 +-
.../operators/index_op/vector_index_query.py | 8 +-
.../operators/kg_construction_task.py | 113 ----
.../operators/llm_op/answer_synthesize.py | 179 ++++--
.../operators/llm_op/disambiguate_data.py | 3 +-
.../operators/llm_op/gremlin_generate.py | 19 +-
.../hugegraph_llm/operators/llm_op/info_extract.py | 68 ++-
.../operators/llm_op/keyword_extract.py | 34 +-
.../operators/llm_op/prompt_generate.py | 6 +-
.../operators/llm_op/property_graph_extract.py | 57 +-
.../hugegraph_llm/operators/llm_op/schema_build.py | 14 +-
.../{graph_rag_task.py => operator_list.py} | 235 ++++----
.../src/hugegraph_llm/{ => state}/__init__.py | 0
hugegraph-llm/src/hugegraph_llm/state/ai_state.py | 279 +++++++++
hugegraph-llm/src/hugegraph_llm/utils/anchor.py | 9 +-
.../src/hugegraph_llm/utils/decorators.py | 1 +
.../src/hugegraph_llm/utils/embedding_utils.py | 9 +-
.../src/hugegraph_llm/utils/graph_index_utils.py | 172 ++----
.../src/hugegraph_llm/utils/hugegraph_utils.py | 26 +-
hugegraph-llm/src/hugegraph_llm/utils/log.py | 2 +-
.../src/hugegraph_llm/utils/vector_index_utils.py | 68 ++-
hugegraph-llm/src/tests/config/test_config.py | 1 +
.../models/embeddings/test_openai_embedding.py | 1 +
.../src/tests/models/llms/test_ollama_client.py | 7 +-
.../tests/operators/common_op/test_check_schema.py | 9 +-
.../tests/operators/common_op/test_nltk_helper.py | 1 +
hugegraph-ml/pyproject.toml | 2 +-
hugegraph-python-client/pyproject.toml | 2 +-
.../src/pyhugegraph/api/auth.py | 16 +-
.../src/pyhugegraph/api/graph.py | 12 +-
.../src/pyhugegraph/api/schema.py | 12 +-
.../pyhugegraph/api/schema_manage/index_label.py | 12 +-
.../src/pyhugegraph/api/services.py | 8 +-
.../src/pyhugegraph/api/traverser.py | 45 +-
hugegraph-python-client/src/pyhugegraph/client.py | 2 +-
.../src/pyhugegraph/example/hugegraph_example.py | 14 +-
.../src/pyhugegraph/structure/property_key_data.py | 4 +-
.../src/pyhugegraph/utils/huge_config.py | 10 +-
.../src/pyhugegraph/utils/huge_router.py | 8 +-
.../src/pyhugegraph/utils/log.py | 4 +-
.../src/pyhugegraph/utils/util.py | 23 +-
hugegraph-python-client/src/tests/api/test_auth.py | 8 +-
.../src/tests/api/test_version.py | 8 +-
hugegraph-python-client/src/tests/client_utils.py | 6 +-
pyproject.toml | 2 +-
scripts/build_llm_image.sh | 2 +-
spec/hugegraph-llm/fixed_flow/design.md | 642 +++++++++++++++++++++
spec/hugegraph-llm/fixed_flow/requirements.md | 24 +
spec/hugegraph-llm/fixed_flow/tasks.md | 36 ++
style/pylint.conf | 4 +-
vermeer-python-client/pyproject.toml | 4 +-
135 files changed, 5690 insertions(+), 1707 deletions(-)
copy hugegraph-llm/src/hugegraph_llm/{enums/id_strategy.py =>
flows/__init__.py} (59%)
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/build_example_index.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/build_schema.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/build_vector_index.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/common.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/flows/get_graph_index_info.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/graph_extract.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/import_graph_data.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/prompt_generate.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/rag_flow_graph_only.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/flows/rag_flow_graph_vector.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/rag_flow_raw.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/flows/rag_flow_vector_only.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/scheduler.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/flows/text2gremlin.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/flows/update_vid_embeddings.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/nodes/base_node.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/common_node/merge_rerank_node.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/document_node/chunk_split.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/hugegraph_node/commit_to_hugegraph.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/hugegraph_node/fetch_graph_data.py
rename
hugegraph-llm/src/hugegraph_llm/{operators/hugegraph_op/graph_rag_query.py =>
nodes/hugegraph_node/graph_query_node.py} (61%)
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/hugegraph_node/gremlin_execute.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/hugegraph_node/schema.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/index_node/build_gremlin_example_index.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/index_node/build_semantic_index.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/index_node/build_vector_index.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/index_node/gremlin_example_index_query.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/index_node/semantic_id_query_node.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/index_node/vector_query_node.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/llm_node/answer_synthesize_node.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/llm_node/extract_info.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/llm_node/keyword_extract_node.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/llm_node/prompt_generate.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/llm_node/schema_build.py
create mode 100644
hugegraph-llm/src/hugegraph_llm/nodes/llm_node/text2gremlin.py
create mode 100644 hugegraph-llm/src/hugegraph_llm/nodes/util.py
delete mode 100644
hugegraph-llm/src/hugegraph_llm/operators/gremlin_generate_task.py
delete mode 100644
hugegraph-llm/src/hugegraph_llm/operators/kg_construction_task.py
rename hugegraph-llm/src/hugegraph_llm/operators/{graph_rag_task.py =>
operator_list.py} (53%)
copy hugegraph-llm/src/hugegraph_llm/{ => state}/__init__.py (100%)
create mode 100644 hugegraph-llm/src/hugegraph_llm/state/ai_state.py
mode change 100644 => 100755 scripts/build_llm_image.sh
create mode 100644 spec/hugegraph-llm/fixed_flow/design.md
create mode 100644 spec/hugegraph-llm/fixed_flow/requirements.md
create mode 100644 spec/hugegraph-llm/fixed_flow/tasks.md
