Copilot commented on code in PR #215:
URL:
https://github.com/apache/incubator-hugegraph-ai/pull/215#discussion_r2063021214
##########
hugegraph-llm/src/hugegraph_llm/operators/index_op/build_semantic_index.py:
##########
@@ -37,34 +39,41 @@ def __init__(self, embedding: BaseEmbedding):
def _extract_names(self, vertices: list[str]) -> list[str]:
return [v.split(":")[1] for v in vertices]
- # TODO: use asyncio for IO tasks
- def _get_embeddings_parallel(self, vids: list[str]) -> list[Any]:
- from concurrent.futures import ThreadPoolExecutor
- with ThreadPoolExecutor() as executor:
- embeddings =
list(tqdm(executor.map(self.embedding.get_text_embedding, vids),
total=len(vids)))
+ async def _get_embeddings_parallel(self, vids: list[str]) -> list[Any]:
+ sem = asyncio.Semaphore(10)
+
+ async def get_embedding_with_semaphore(vid: str) -> Any:
+ async with sem:
+ loop = asyncio.get_running_loop()
+ return await loop.run_in_executor(None,
self.embedding.get_text_embedding, vid)
+
+ tasks = [get_embedding_with_semaphore(vid) for vid in vids]
+ embeddings = []
+ with tqdm(total=len(tasks)) as pbar:
+ for future in asyncio.as_completed(tasks):
+ embedding = await future
Review Comment:
Using asyncio.as_completed returns results in the order of task completion
rather than the original order of vids. If the correspondence between
embeddings and vids matters, consider using 'asyncio.gather' to preserve task
ordering.
```suggestion
for embedding in await asyncio.gather(*tasks):
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
