This is an automated email from the ASF dual-hosted git repository.
jin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-ai.git
The following commit(s) were added to refs/heads/main by this push:
new 40c285f feat(llm): support any openai-style api(standard) (#95)
40c285f is described below
commit 40c285f24a69e5e2fc96eadd7bc33592b30bbd7a
Author: chenzihong <[email protected]>
AuthorDate: Mon Oct 21 15:55:57 2024 +0800
feat(llm): support any openai-style api(standard) (#95)
Make some change to support a standard openai-style api.
Now we can use LLMs and Embedding models supported by other companies.
Also, I changed ways to test llm and embedding connection.
Before, we request `/model`, which is not always reliable. e.g. if
`/chat/completions` fails down but `/model` still alive.
---
.../src/hugegraph_llm/config/config_data.py | 2 ++
.../hugegraph_llm/demo/rag_demo/configs_block.py | 23 ++++++++++++++--------
.../models/embeddings/init_embedding.py | 4 ++--
3 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/hugegraph-llm/src/hugegraph_llm/config/config_data.py
b/hugegraph-llm/src/hugegraph_llm/config/config_data.py
index 0742190..3610aa2 100644
--- a/hugegraph-llm/src/hugegraph_llm/config/config_data.py
+++ b/hugegraph-llm/src/hugegraph_llm/config/config_data.py
@@ -33,6 +33,8 @@ class ConfigData:
openai_api_base: Optional[str] = os.environ.get("OPENAI_BASE_URL",
"https://api.openai.com/v1";)
openai_api_key: Optional[str] = os.environ.get("OPENAI_API_KEY")
openai_language_model: Optional[str] = "gpt-4o-mini"
+ openai_embedding_api_base: Optional[str] =
os.environ.get("OPENAI_EMBEDDING_BASE_URL", "https://api.openai.com/v1";)
+ openai_embedding_api_key: Optional[str] =
os.environ.get("OPENAI_EMBEDDING_API_KEY")
openai_embedding_model: Optional[str] = "text-embedding-3-small"
openai_max_tokens: int = 4096
# 2. Rerank settings
diff --git a/hugegraph-llm/src/hugegraph_llm/demo/rag_demo/configs_block.py
b/hugegraph-llm/src/hugegraph_llm/demo/rag_demo/configs_block.py
index 824f41a..cf3e139 100644
--- a/hugegraph-llm/src/hugegraph_llm/demo/rag_demo/configs_block.py
+++ b/hugegraph-llm/src/hugegraph_llm/demo/rag_demo/configs_block.py
@@ -80,12 +80,13 @@ def apply_embedding_config(arg1, arg2, arg3,
origin_call=None) -> int:
status_code = -1
embedding_option = settings.embedding_type
if embedding_option == "openai":
- settings.openai_api_key = arg1
- settings.openai_api_base = arg2
+ settings.openai_embedding_api_key = arg1
+ settings.openai_embedding_api_base = arg2
settings.openai_embedding_model = arg3
- test_url = settings.openai_api_base + "/models"
+ test_url = settings.openai_embedding_api_base + "/embeddings"
headers = {"Authorization": f"Bearer {arg1}"}
- status_code = test_api_connection(test_url, headers=headers,
origin_call=origin_call)
+ data = {"model": arg3, "input": "test"}
+ status_code = test_api_connection(test_url, method="POST",
headers=headers, body=data, origin_call=origin_call)
elif embedding_option == "qianfan_wenxin":
status_code = config_qianfan_model(arg1, arg2, origin_call=origin_call)
settings.qianfan_embedding_model = arg3
@@ -165,9 +166,14 @@ def apply_llm_config(arg1, arg2, arg3, arg4,
origin_call=None) -> int:
settings.openai_api_base = arg2
settings.openai_language_model = arg3
settings.openai_max_tokens = int(arg4)
- test_url = settings.openai_api_base + "/models"
+ test_url = settings.openai_api_base + "/chat/completions"
+ data = {
+ "model": arg3,
+ "temperature": 0.0,
+ "messages": [{"role": "user", "content": "test"}],
+ }
headers = {"Authorization": f"Bearer {arg1}"}
- status_code = test_api_connection(test_url, headers=headers,
origin_call=origin_call)
+ status_code = test_api_connection(test_url, method="POST",
headers=headers, body=data, origin_call=origin_call)
elif llm_option == "qianfan_wenxin":
status_code = config_qianfan_model(arg1, arg2, arg3, origin_call)
elif llm_option == "ollama":
@@ -196,6 +202,7 @@ def create_configs_block() -> list:
graph_config_button.click(apply_graph_config, inputs=graph_config_input)
# pylint: disable=no-member
with gr.Accordion("2. Set up the LLM.", open=False):
+ gr.Markdown("> Tips: the openai sdk also support openai style api from
other providers.")
llm_dropdown = gr.Dropdown(choices=["openai", "qianfan_wenxin",
"ollama"], value=settings.llm_type, label="LLM")
@gr.render(inputs=[llm_dropdown])
@@ -247,8 +254,8 @@ def create_configs_block() -> list:
if embedding_type == "openai":
with gr.Row():
embedding_config_input = [
- gr.Textbox(value=settings.openai_api_key,
label="api_key", type="password"),
- gr.Textbox(value=settings.openai_api_base,
label="api_base"),
+ gr.Textbox(value=settings.openai_embedding_api_key,
label="api_key", type="password"),
+ gr.Textbox(value=settings.openai_embedding_api_base,
label="api_base"),
gr.Textbox(value=settings.openai_embedding_model,
label="model_name"),
]
elif embedding_type == "ollama":
diff --git
a/hugegraph-llm/src/hugegraph_llm/models/embeddings/init_embedding.py
b/hugegraph-llm/src/hugegraph_llm/models/embeddings/init_embedding.py
index c8004a6..ded48af 100644
--- a/hugegraph-llm/src/hugegraph_llm/models/embeddings/init_embedding.py
+++ b/hugegraph-llm/src/hugegraph_llm/models/embeddings/init_embedding.py
@@ -30,8 +30,8 @@ class Embeddings:
if self.embedding_type == "openai":
return OpenAIEmbedding(
model_name=settings.openai_embedding_model,
- api_key=settings.openai_api_key,
- api_base=settings.openai_api_base,
+ api_key=settings.openai_embedding_api_key,
+ api_base=settings.openai_embedding_api_base
)
if self.embedding_type == "ollama":
return OllamaEmbedding(
