This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 8596879c docs: add Flight SQL driver recipes (#1547)
8596879c is described below
commit 8596879cc1d2451fb2c7aea99b42a0b484f484f5
Author: David Li <[email protected]>
AuthorDate: Tue Feb 13 16:10:50 2024 -0500
docs: add Flight SQL driver recipes (#1547)
Fixes #935.
---
.github/workflows/native-unix.yml | 48 ++++++------------
ci/scripts/python_build.sh | 12 ++++-
docker-compose.yml | 6 ++-
docs/source/driver/flight_sql.rst | 2 +
docs/source/python/recipe/flight_sql.rst | 58 ++++++++++++++++++++++
.../python/recipe/flightsql_dremio_connect.py | 45 ++++++++---------
.../python/recipe/flightsql_sqlite_connect.py | 35 +++++--------
.../python/recipe/flightsql_sqlite_options.py | 56 +++++++++++++++++++++
docs/source/python/recipe/index.rst | 1 +
9 files changed, 183 insertions(+), 80 deletions(-)
diff --git a/.github/workflows/native-unix.yml
b/.github/workflows/native-unix.yml
index 2432fb40..c42b56aa 100644
--- a/.github/workflows/native-unix.yml
+++ b/.github/workflows/native-unix.yml
@@ -451,10 +451,14 @@ jobs:
name: driver-manager-${{ matrix.os }}
path: ~/local
- - name: Build Python Driver Manager
+ - name: Build
shell: bash -l {0}
+ env:
+ BUILD_ALL: "1"
+ ADBC_USE_ASAN: "0"
+ ADBC_USE_UBSAN: "0"
run: |
- env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 ./ci/scripts/python_build.sh
"$(pwd)" "$(pwd)/build" "$HOME/local"
+ ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Build Panic Dummy
shell: bash -l {0}
run: |
@@ -472,10 +476,6 @@ jobs:
export
PANICDUMMY_LIBRARY_PATH=$(pwd)/go/adbc/pkg/libadbc_driver_panicdummy.so
fi
env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 ./ci/scripts/python_test.sh
"$(pwd)" "$(pwd)/build" "$HOME/local"
- - name: Build Python Driver Flight SQL
- shell: bash -l {0}
- run: |
- env BUILD_ALL=0 BUILD_DRIVER_FLIGHTSQL=1
./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Test Python Driver Flight SQL
shell: bash -l {0}
run: |
@@ -493,26 +493,14 @@ jobs:
export ADBC_TEST_FLIGHTSQL_URI=grpc://localhost:41414
env BUILD_ALL=0 BUILD_DRIVER_FLIGHTSQL=1 ./ci/scripts/python_test.sh
"$(pwd)" "$(pwd)/build" "$HOME/local"
kill %1
- - name: Build Python Driver PostgreSQL
- shell: bash -l {0}
- run: |
- env BUILD_ALL=0 BUILD_DRIVER_POSTGRESQL=1
./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Test Python Driver PostgreSQL
shell: bash -l {0}
run: |
env BUILD_ALL=0 BUILD_DRIVER_POSTGRESQL=1
./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- - name: Build Python Driver SQLite
- shell: bash -l {0}
- run: |
- env BUILD_ALL=0 BUILD_DRIVER_SQLITE=1 ./ci/scripts/python_build.sh
"$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Test Python Driver SQLite
shell: bash -l {0}
run: |
env BUILD_ALL=0 BUILD_DRIVER_SQLITE=1 ./ci/scripts/python_test.sh
"$(pwd)" "$(pwd)/build" "$HOME/local"
- - name: Build Python Driver Snowflake
- shell: bash -l {0}
- run: |
- env BUILD_ALL=0 BUILD_DRIVER_SNOWFLAKE=1
./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Test Python Driver Snowflake
shell: bash -l {0}
run: |
@@ -527,19 +515,6 @@ jobs:
runs-on: ${{ matrix.os }}
needs:
- drivers-build-conda
- services:
- postgres:
- image: postgres
- env:
- POSTGRES_DB: postgres
- POSTGRES_PASSWORD: password
- options: >-
- --health-cmd pg_isready
- --health-interval 10s
- --health-timeout 5s
- --health-retries 5
- ports:
- - 5432:5432
strategy:
matrix:
os: ["ubuntu-latest"]
@@ -579,8 +554,12 @@ jobs:
- name: Build Python
shell: bash -l {0}
+ env:
+ BUILD_ALL: "1"
+ ADBC_USE_ASAN: "0"
+ ADBC_USE_UBSAN: "0"
run: |
- env BUILD_ALL=1 ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
"$HOME/local"
+ ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
# Docs requires Python packages since it runs doctests
- name: Build Docs
shell: bash -l {0}
@@ -601,5 +580,10 @@ jobs:
shell: bash -l {0}
env:
ADBC_POSTGRESQL_TEST_URI:
"postgres://localhost:5432/postgres?user=postgres&password=password"
+ ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010"
+ ADBC_DREMIO_FLIGHTSQL_USER: "dremio"
+ ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123"
+ ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080"
run: |
+ docker compose up --detach --wait dremio dremio-init
flightsql-sqlite-test postgres-test
pytest -vvs docs/source/tests/
diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh
index 533a82a3..76f125f5 100755
--- a/ci/scripts/python_build.sh
+++ b/ci/scripts/python_build.sh
@@ -25,6 +25,11 @@ set -e
: ${BUILD_DRIVER_SQLITE:=${BUILD_ALL}}
: ${BUILD_DRIVER_SNOWFLAKE:=${BUILD_ALL}}
+: ${ADBC_USE_ASAN:=ON}
+: ${ADBC_USE_UBSAN:=ON}
+
+: ${ADBC_CMAKE_ARGS:=""}
+: ${CMAKE_BUILD_TYPE:=Debug}
main() {
local -r source_dir="${1}"
@@ -32,7 +37,12 @@ main() {
set -x
- cmake -S "${source_dir}/c" -B ${build_dir} \
+ cmake -S "${source_dir}/c" \
+ -B ${build_dir} \
+ ${ADBC_CMAKE_ARGS} \
+ -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \
+ -DADBC_USE_ASAN="${ADBC_USE_ASAN}" \
+ -DADBC_USE_UBSAN="${ADBC_USE_UBSAN}" \
-DADBC_DRIVER_MANAGER=${BUILD_DRIVER_MANAGER} \
-DADBC_DRIVER_FLIGHTSQL=${BUILD_DRIVER_FLIGHTSQL} \
-DADBC_DRIVER_POSTGRESQL=${BUILD_DRIVER_POSTGRESQL} \
diff --git a/docker-compose.yml b/docker-compose.yml
index 789d5d45..d4113e8f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -27,8 +27,12 @@ services:
image: condaforge/mambaforge:latest
volumes:
- .:/adbc:delegated
+ environment:
+ ADBC_USE_ASAN: "0"
+ ADBC_USE_UBSAN: "0"
+ CGO_ENABLED: "1"
command: |
- /bin/bash -c 'git config --global --add safe.directory /adbc && source
/opt/conda/etc/profile.d/conda.sh && mamba create -y -n adbc -c conda-forge go
--file /adbc/ci/conda_env_cpp.txt --file /adbc/ci/conda_env_docs.txt --file
/adbc/ci/conda_env_java.txt --file /adbc/ci/conda_env_python.txt && conda
activate adbc && env ADBC_USE_ASAN=0 ADBC_USE_UBSAN=0
/adbc/ci/scripts/cpp_build.sh /adbc /adbc/build && env CGO_ENABLED=1
/adbc/ci/scripts/go_build.sh /adbc /adbc/build && /adbc/ci/scri [...]
+ /bin/bash -c 'git config --global --add safe.directory /adbc && source
/opt/conda/etc/profile.d/conda.sh && mamba create -y -n adbc -c conda-forge go
--file /adbc/ci/conda_env_cpp.txt --file /adbc/ci/conda_env_docs.txt --file
/adbc/ci/conda_env_java.txt --file /adbc/ci/conda_env_python.txt && conda
activate adbc && /adbc/ci/scripts/cpp_build.sh /adbc /adbc/build &&
/adbc/ci/scripts/go_build.sh /adbc /adbc/build &&
/adbc/ci/scripts/python_build.sh /adbc /adbc/build && /adbc/ci/scrip [...]
############################ Java JARs ######################################
diff --git a/docs/source/driver/flight_sql.rst
b/docs/source/driver/flight_sql.rst
index a9067fcb..add9ea41 100644
--- a/docs/source/driver/flight_sql.rst
+++ b/docs/source/driver/flight_sql.rst
@@ -123,6 +123,8 @@ the :cpp:class:`AdbcDatabase`.
) as conn:
pass
+ For more examples, see :doc:`../python/recipe/flight_sql`.
+
.. tab-item:: Go
:sync: go
diff --git a/docs/source/python/recipe/flight_sql.rst
b/docs/source/python/recipe/flight_sql.rst
new file mode 100644
index 00000000..c3bffb13
--- /dev/null
+++ b/docs/source/python/recipe/flight_sql.rst
@@ -0,0 +1,58 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+==================
+Flight SQL Recipes
+==================
+
+Some of these recipes are written againt a demo Flight SQL service backed by
+SQLite. You can run it yourself as follows:
+
+.. code-block:: shell
+
+ $ go install
github.com/apache/arrow/go/v${ARROW_MAJOR_VERSION}/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server@latest
+ $ sqlite_flightsql_server -host 0.0.0.0 -port 8080
+
+Other recipes work using the OSS version of Dremio_:
+
+.. code-block:: shell
+
+ $ docker run -p 9047:9047 -p 31010:31010 -p 45678:45678 dremio/dremio-oss
+
+If you have the ADBC repository checked out and Docker Compose installed, you
+can use our configuration to run both services:
+
+.. code-block:: shell
+
+ $ docker compose up --detach --wait dremio dremio-init flightsql-sqlite-test
+
+.. _Dremio: https://www.dremio.com/
+
+Connect to an unsecured Flight SQL service
+------------------------------------------
+
+.. recipe:: flightsql_sqlite_connect.py
+
+Connect to a Flight SQL service with username and password
+----------------------------------------------------------
+
+.. recipe:: flightsql_dremio_connect.py
+
+Set timeouts and other options
+------------------------------
+
+.. recipe:: flightsql_sqlite_options.py
diff --git a/ci/scripts/python_build.sh
b/docs/source/python/recipe/flightsql_dremio_connect.py
old mode 100755
new mode 100644
similarity index 50%
copy from ci/scripts/python_build.sh
copy to docs/source/python/recipe/flightsql_dremio_connect.py
index 533a82a3..9ba5a9e9
--- a/ci/scripts/python_build.sh
+++ b/docs/source/python/recipe/flightsql_dremio_connect.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,32 +15,32 @@
# specific language governing permissions and limitations
# under the License.
-set -e
+# RECIPE STARTS HERE
-: ${BUILD_ALL:=1}
-: ${BUILD_DRIVER_FLIGHTSQL:=${BUILD_ALL}}
-: ${BUILD_DRIVER_MANAGER:=${BUILD_ALL}}
-: ${BUILD_DRIVER_POSTGRESQL:=${BUILD_ALL}}
-: ${BUILD_DRIVER_SQLITE:=${BUILD_ALL}}
-: ${BUILD_DRIVER_SNOWFLAKE:=${BUILD_ALL}}
+#: Dremio requires a username and password. To connect to a Flight SQL
+#: service with authentication, provide the options at connection time.
+import os
-main() {
- local -r source_dir="${1}"
- local -r build_dir="${2}"
+import adbc_driver_flightsql.dbapi
+import adbc_driver_manager
- set -x
+uri = os.environ["ADBC_DREMIO_FLIGHTSQL_URI"]
+username = os.environ["ADBC_DREMIO_FLIGHTSQL_USER"]
+password = os.environ["ADBC_DREMIO_FLIGHTSQL_PASS"]
+conn = adbc_driver_flightsql.dbapi.connect(
+ uri,
+ db_kwargs={
+ adbc_driver_manager.DatabaseOptions.USERNAME.value: username,
+ adbc_driver_manager.DatabaseOptions.PASSWORD.value: password,
+ },
+)
- cmake -S "${source_dir}/c" -B ${build_dir} \
- -DADBC_DRIVER_MANAGER=${BUILD_DRIVER_MANAGER} \
- -DADBC_DRIVER_FLIGHTSQL=${BUILD_DRIVER_FLIGHTSQL} \
- -DADBC_DRIVER_POSTGRESQL=${BUILD_DRIVER_POSTGRESQL} \
- -DADBC_DRIVER_SQLITE=${BUILD_DRIVER_SQLITE} \
- -DADBC_DRIVER_SNOWFLAKE=${BUILD_DRIVER_SNOWFLAKE} \
- -DADBC_BUILD_PYTHON=ON
- cmake --build ${build_dir} --target python
+#: We can then execute a simple query.
- set +x
-}
+with conn.cursor() as cur:
+ cur.execute("SELECT 1")
-main "$@"
+ assert cur.fetchone() == (1,)
+
+conn.close()
diff --git a/ci/scripts/python_build.sh
b/docs/source/python/recipe/flightsql_sqlite_connect.py
old mode 100755
new mode 100644
similarity index 50%
copy from ci/scripts/python_build.sh
copy to docs/source/python/recipe/flightsql_sqlite_connect.py
index 533a82a3..97148e6d
--- a/ci/scripts/python_build.sh
+++ b/docs/source/python/recipe/flightsql_sqlite_connect.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,32 +15,22 @@
# specific language governing permissions and limitations
# under the License.
-set -e
+# RECIPE STARTS HERE
-: ${BUILD_ALL:=1}
-: ${BUILD_DRIVER_FLIGHTSQL:=${BUILD_ALL}}
-: ${BUILD_DRIVER_MANAGER:=${BUILD_ALL}}
-: ${BUILD_DRIVER_POSTGRESQL:=${BUILD_ALL}}
-: ${BUILD_DRIVER_SQLITE:=${BUILD_ALL}}
-: ${BUILD_DRIVER_SNOWFLAKE:=${BUILD_ALL}}
+#: To connect to an unsecured Flight SQL service, just provide the URI.
+import os
-main() {
- local -r source_dir="${1}"
- local -r build_dir="${2}"
+import adbc_driver_flightsql.dbapi
- set -x
+uri = os.environ["ADBC_SQLITE_FLIGHTSQL_URI"]
+conn = adbc_driver_flightsql.dbapi.connect(uri)
- cmake -S "${source_dir}/c" -B ${build_dir} \
- -DADBC_DRIVER_MANAGER=${BUILD_DRIVER_MANAGER} \
- -DADBC_DRIVER_FLIGHTSQL=${BUILD_DRIVER_FLIGHTSQL} \
- -DADBC_DRIVER_POSTGRESQL=${BUILD_DRIVER_POSTGRESQL} \
- -DADBC_DRIVER_SQLITE=${BUILD_DRIVER_SQLITE} \
- -DADBC_DRIVER_SNOWFLAKE=${BUILD_DRIVER_SNOWFLAKE} \
- -DADBC_BUILD_PYTHON=ON
- cmake --build ${build_dir} --target python
+#: We can then execute a simple query.
- set +x
-}
+with conn.cursor() as cur:
+ cur.execute("SELECT 1")
-main "$@"
+ assert cur.fetchone() == (1,)
+
+conn.close()
diff --git a/docs/source/python/recipe/flightsql_sqlite_options.py
b/docs/source/python/recipe/flightsql_sqlite_options.py
new file mode 100644
index 00000000..4dee1e61
--- /dev/null
+++ b/docs/source/python/recipe/flightsql_sqlite_options.py
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# RECIPE STARTS HERE
+
+#: The Flight SQL driver supports various options.
+
+import os
+
+import adbc_driver_flightsql.dbapi
+from adbc_driver_flightsql import ConnectionOptions, DatabaseOptions
+
+uri = os.environ["ADBC_SQLITE_FLIGHTSQL_URI"]
+#: We can enable cookie support, which some server implementations require.
+conn = adbc_driver_flightsql.dbapi.connect(
+ uri,
+ db_kwargs={DatabaseOptions.WITH_COOKIE_MIDDLEWARE.value: "true"},
+)
+
+#: Other options are set on the connection or statement.
+
+#: For example, we can add a custom header to all outgoing requests.
+custom_header =
f"{ConnectionOptions.RPC_CALL_HEADER_PREFIX.value}x-custom-header"
+conn.adbc_connection.set_options(**{custom_header: "value"})
+
+#: We can also set timeouts. These are in floating-point seconds.
+conn.adbc_connection.set_options(
+ **{
+ ConnectionOptions.TIMEOUT_FETCH.value: 30.0,
+ ConnectionOptions.TIMEOUT_QUERY.value: 30.0,
+ ConnectionOptions.TIMEOUT_UPDATE.value: 30.0,
+ }
+)
+
+#: These options will apply to all cursors we create.
+
+with conn.cursor() as cur:
+ cur.execute("SELECT 1")
+
+ assert cur.fetchone() == (1,)
+
+conn.close()
diff --git a/docs/source/python/recipe/index.rst
b/docs/source/python/recipe/index.rst
index b841b7f1..3645e0aa 100644
--- a/docs/source/python/recipe/index.rst
+++ b/docs/source/python/recipe/index.rst
@@ -26,4 +26,5 @@ Python.
:maxdepth: 2
driver_manager
+ flight_sql
postgresql
