This is an automated email from the ASF dual-hosted git repository.

pankaj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 421703c741 Adding Pgvector system tests (#36592)
421703c741 is described below

commit 421703c741ad5d0cd3ab2b2814f3cea3f51afb2b
Author: vatsrahul1001 <[email protected]>
AuthorDate: Thu Jan 4 22:18:11 2024 +0530

    Adding Pgvector system tests (#36592)
    
    * add pgvector dags
    
    * add pgvector dags
    
    * remove not used tag
---
 .../providers/pgvector/example_pgvector_openai.py  | 96 ++++++++++++++++++++++
 1 file changed, 96 insertions(+)

diff --git a/tests/system/providers/pgvector/example_pgvector_openai.py 
b/tests/system/providers/pgvector/example_pgvector_openai.py
new file mode 100644
index 0000000000..d1379785f9
--- /dev/null
+++ b/tests/system/providers/pgvector/example_pgvector_openai.py
@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pendulum
+
+from airflow.decorators import dag, task, teardown
+from airflow.providers.openai.operators.openai import OpenAIEmbeddingOperator
+from airflow.providers.pgvector.operators.pgvector import 
PgVectorIngestOperator
+
+TABLE_NAME = "my_table"
+POSTGRES_CONN_ID = "postgres_default"
+
+
+@dag(
+    "example_openai_pgvector_dag",
+    schedule_interval=None,
+    start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
+    catchup=False,
+    tags=["example", "pgvector"],
+)
+def example_pgvector_dag():
+    """Example pgvector DAG demonstrating usage of the 
PgVectorIngestOperator."""
+
+    @task()
+    def create_postgres_objects():
+        """
+        Example task to create PostgreSQL objects including table and 
installing the vector extension using
+        the PgVectorHook.
+        """
+        from airflow.providers.pgvector.hooks.pgvector import PgVectorHook
+
+        pg_hook = PgVectorHook(postgres_conn_id=POSTGRES_CONN_ID)
+
+        # Create a table
+        columns = [
+            "id SERIAL PRIMARY KEY",
+            "name VARCHAR(255)",
+            "value INTEGER",
+            "vector_column vector(1536)",
+        ]
+        pg_hook.create_table(TABLE_NAME, columns)
+
+        # Create vector extension
+        extension_name = "vector"
+        pg_hook.create_extension(extension_name)
+
+    embedd_query = OpenAIEmbeddingOperator(
+        task_id="embedding_using_xcom_data",
+        conn_id="openai_default",
+        input_text="123",
+        model="text-embedding-ada-002",
+    )
+
+    pgvector_ingest = PgVectorIngestOperator(
+        task_id="pgvector_ingest",
+        conn_id=POSTGRES_CONN_ID,
+        sql=f"INSERT INTO {TABLE_NAME} (name, value, vector_column) "
+        f"VALUES ('John Doe', '123' ,'{embedd_query.output}')",
+    )
+
+    @teardown()
+    @task()
+    def cleanup_postgres_objects():
+        """
+        Cleanup Postgres objects created in the earlier task.
+        """
+        from airflow.providers.pgvector.hooks.pgvector import PgVectorHook
+
+        pg_hook = PgVectorHook(postgres_conn_id=POSTGRES_CONN_ID)
+        pg_hook.truncate_table(TABLE_NAME)
+        pg_hook.drop_table(TABLE_NAME)
+
+    create_postgres_objects() >> pgvector_ingest >> cleanup_postgres_objects()
+
+
+example_pgvector_dag()
+
+from tests.system.utils import get_test_run  # noqa: E402
+
+# Needed to run the example DAG with pytest (see: 
tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)

Reply via email to