This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new abf67e92 ci(r): Refactor CI configuration (#1210)
abf67e92 is described below
commit abf67e922963adeeada4e0f1fcbabb57ab5e264b
Author: Dewey Dunnington <[email protected]>
AuthorDate: Fri Oct 20 11:34:30 2023 -0300
ci(r): Refactor CI configuration (#1210)
This PR moves R workflows into their own file since they don't depend on
any other jobs. It also adds an r-extended job that checks some of the
more obscure things (e.g., older R versions, valgrind). The gist of the
CI setup is:
- r-basic.yml: Makes sure that drivers build + pass tests with test
databases on Linux. These run when C, R, or Go code is modified. This
catches changes to the drivers.
- r-standard.yml: Checks driver manager and driver packages on Mac,
Windows, and Linux. These run when working on the R packages because
that is the time that MacOS and Windows-specific regressions are likely
to occur (e.g., when working on the build systems).
- r-extended.yml: Checks the way CRAN would check (on oldrel, release,
and devel versions of R on Mac, Windows, and Linux); checks older R
versions still in the tidyverse support grid on Windows (because the
build system changes so frequently) and Linux (because Go drivers don't
build on older R versions on Windows); runs tests with valgrind.
Closes #1138.
I opened tickets to debug the valgrind errors reported by the drivers.
Those CI jobs run weekly (reflecting the time I have available to debug
them) so they won't add noise to ongoing development.
---
.github/workflows/native-unix.yml | 105 ---------------------------
.github/workflows/r-basic.yml | 53 ++++++++++++++
.github/workflows/r-check.yml | 87 ++++++++++++++++++++++
.github/workflows/r-extended.yml | 147 ++++++++++++++++++++++++++++++++++++++
.github/workflows/r-standard.yml | 62 ++++++++++++++++
r/valgrind.supp | 23 ++++++
6 files changed, 372 insertions(+), 105 deletions(-)
diff --git a/.github/workflows/native-unix.yml
b/.github/workflows/native-unix.yml
index a88eb9c5..87e13682 100644
--- a/.github/workflows/native-unix.yml
+++ b/.github/workflows/native-unix.yml
@@ -29,7 +29,6 @@ on:
- "glib/**"
- "go/**"
- "python/**"
- - "r/**"
- "ruby/**"
- ".github/workflows/native-unix.yml"
push:
@@ -41,7 +40,6 @@ on:
- "glib/**"
- "go/**"
- "python/**"
- - "r/**"
- "ruby/**"
- ".github/workflows/native-unix.yml"
@@ -583,106 +581,3 @@ jobs:
ADBC_POSTGRESQL_TEST_URI:
"postgres://localhost:5432/postgres?user=postgres&password=password"
run: |
pytest -vvs docs/source/tests/
-
- # ------------------------------------------------------------
- # R
- # ------------------------------------------------------------
- r:
- name: "R-${{ matrix.config.r }}/${{ matrix.config.pkg }} (${{
matrix.config.os }})"
- runs-on: ${{ matrix.config.os }}
- strategy:
- matrix:
- config:
- - {os: macOS-latest, r: 'release', pkg: 'adbcdrivermanager'}
- - {os: windows-latest, r: 'release', pkg: 'adbcdrivermanager'}
- - {os: ubuntu-latest, r: 'release', pkg: 'adbcdrivermanager'}
- - {os: macOS-latest, r: 'release', pkg: 'adbcsqlite'}
- - {os: windows-latest, r: 'release', pkg: 'adbcsqlite'}
- - {os: ubuntu-latest, r: 'release', pkg: 'adbcsqlite'}
- - {os: macOS-latest, r: 'release', pkg: 'adbcpostgresql'}
- - {os: windows-latest, r: 'release', pkg: 'adbcpostgresql'}
- - {os: windows-latest, r: '3.6', pkg: 'adbcpostgresql'}
- - {os: ubuntu-latest, r: 'release', pkg: 'adbcpostgresql'}
- - {os: macOS-latest, r: 'release', pkg: 'adbcsnowflake'}
- - {os: windows-latest, r: 'release', pkg: 'adbcsnowflake'}
- - {os: ubuntu-latest, r: 'release', pkg: 'adbcsnowflake'}
- - {os: ubuntu-latest, r: 'release', pkg: 'adbcflightsql'}
- - {os: macOS-latest, r: 'release', pkg: 'adbcflightsql'}
- - {os: windows-latest, r: 'release', pkg: 'adbcflightsql'}
-
- env:
- GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
- R_KEEP_PKG_SOURCE: yes
-
- steps:
- - uses: actions/checkout@v3
- with:
- fetch-depth: 0
- persist-credentials: false
-
- - uses: r-lib/actions/setup-pandoc@v2
- - uses: r-lib/actions/setup-r@v2
- with:
- r-version: ${{ matrix.config.r }}
- http-user-agent: ${{ matrix.config.http-user-agent }}
- use-public-rspm: true
-
- - name: Set PKG_CONFIG_PATH on MacOS
- if: matrix.config.pkg == 'adbcpostgresql' && runner.os == 'macOS'
- run: |
- PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$(brew --prefix
libpq)/lib/pkgconfig:$(brew --prefix openssl)/lib/pkgconfig"
- echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" >> $GITHUB_ENV
-
- - name: Prepare sources (driver manager)
- if: matrix.config.pkg == 'adbcdrivermanager'
- run: |
- R -e "install.packages('nanoarrow', repos =
'https://cloud.r-project.org')"
- R CMD INSTALL r/${{ matrix.config.pkg }}
- shell: bash
-
- - name: Prepare sources
- if: matrix.config.pkg != 'adbcdrivermanager'
- run: |
- R -e "install.packages('nanoarrow', repos =
'https://cloud.r-project.org')"
- R CMD INSTALL r/adbcdrivermanager
- R CMD INSTALL r/${{ matrix.config.pkg }}
- shell: bash
-
- - uses: r-lib/actions/setup-r-dependencies@v2
- with:
- extra-packages: any::rcmdcheck, local::../adbcdrivermanager
- needs: check
- working-directory: r/${{ matrix.config.pkg }}
-
- - name: Start postgres test database
- if: matrix.config.pkg == 'adbcpostgresql' && runner.os == 'Linux'
- run: |
- cd r/adbcpostgresql
- docker compose up --detach postgres-test
-
ADBC_POSTGRESQL_TEST_URI="postgresql://localhost:5432/postgres?user=postgres&password=password"
- echo "ADBC_POSTGRESQL_TEST_URI=${ADBC_POSTGRESQL_TEST_URI}" >>
$GITHUB_ENV
-
- - name: Start FlightSQL test database
- if: matrix.config.pkg == 'adbcflightsql' && runner.os == 'Linux'
- run: |
- cd r/adbcpostgresql
- docker compose up --detach flightsql-sqlite-test
- ADBC_FLIGHTSQL_TEST_URI="grpc://localhost:8080"
- echo "ADBC_FLIGHTSQL_TEST_URI=${ADBC_FLIGHTSQL_TEST_URI}" >>
$GITHUB_ENV
-
- - uses: r-lib/actions/check-r-package@v2
- env:
- ADBC_SNOWFLAKE_TEST_URI: ${{ secrets.SNOWFLAKE_URI }}
- with:
- upload-snapshots: true
- working-directory: r/${{ matrix.config.pkg }}
- # Using --no-multiarch here means that in version of R that support
- # 32-bit Windows builds, only the 64-bit version is tested. With some
- # effort we could also test 32-bit builds; however, they are not
commonly
- # used.
- args: 'c("--no-manual", "--as-cran", "--no-multiarch")'
-
- - name: Stop test database
- if: runner.os == 'Linux'
- run: |
- docker compose down
diff --git a/.github/workflows/r-basic.yml b/.github/workflows/r-basic.yml
new file mode 100644
index 00000000..e698b34b
--- /dev/null
+++ b/.github/workflows/r-basic.yml
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: R
+
+# Runs on every PR that touches something the R package might use
+on:
+ pull_request:
+ branches:
+ - main
+ paths:
+ - "adbc.h"
+ - "c/**"
+ - "go/adbc/driver/**"
+ - "go/adbc/pkg/**"
+ - "r/**"
+ - ".github/workflows/r-basic.yml"
+ - ".github/workflows/r-check.yml"
+
+concurrency:
+ group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ check:
+ strategy:
+ matrix:
+ os: [ubuntu]
+ pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql, adbcflightsql,
adbcsnowflake]
+
+ uses: ./.github/workflows/r-check.yml
+ with:
+ os: ${{ matrix.os }}
+ pkg: ${{ matrix.pkg }}
+ secrets:
+ SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
diff --git a/.github/workflows/r-check.yml b/.github/workflows/r-check.yml
new file mode 100644
index 00000000..b8373a9d
--- /dev/null
+++ b/.github/workflows/r-check.yml
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Reusable check for various combinations of (pkg, os, rversion)
+name: R
+
+on:
+ workflow_call:
+ inputs:
+ os:
+ required: true
+ type: string
+ pkg:
+ required: true
+ type: string
+ rversion:
+ required: false
+ default: release
+ type: string
+ secrets:
+ SNOWFLAKE_URI:
+ required: false
+
+jobs:
+ r-check:
+ runs-on: ${{ inputs.os }}-latest
+
+ steps:
+ - uses: actions/checkout@v4
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ r-version: release
+ use-public-rspm: true
+
+ - name: Set PKG_CONFIG_PATH on MacOS
+ if: inputs.pkg == 'adbcpostgresql' && runner.os == 'macOS'
+ run: |
+ PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$(brew --prefix
libpq)/lib/pkgconfig:$(brew --prefix openssl)/lib/pkgconfig"
+ echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" >> $GITHUB_ENV
+
+ - uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ extra-packages: any::rcmdcheck, local::../adbcdrivermanager
+ needs: check
+ working-directory: r/${{ inputs.pkg }}
+
+ - name: Start postgres test database
+ if: inputs.pkg == 'adbcpostgresql' && runner.os == 'Linux'
+ run: |
+ cd r/adbcpostgresql
+ docker compose up --detach postgres-test
+
ADBC_POSTGRESQL_TEST_URI="postgresql://localhost:5432/postgres?user=postgres&password=password"
+ echo "ADBC_POSTGRESQL_TEST_URI=${ADBC_POSTGRESQL_TEST_URI}" >>
$GITHUB_ENV
+
+ - name: Start FlightSQL test database
+ if: inputs.pkg == 'adbcflightsql' && runner.os == 'Linux'
+ run: |
+ cd r/adbcpostgresql
+ docker compose up --detach flightsql-sqlite-test
+ ADBC_FLIGHTSQL_TEST_URI="grpc://localhost:8080"
+ echo "ADBC_FLIGHTSQL_TEST_URI=${ADBC_FLIGHTSQL_TEST_URI}" >>
$GITHUB_ENV
+
+ - uses: r-lib/actions/check-r-package@v2
+ env:
+ ADBC_SNOWFLAKE_TEST_URI: ${{ secrets.SNOWFLAKE_URI }}
+ R_KEEP_PKG_SOURCE: yes
+ with:
+ working-directory: r/${{ inputs.pkg }}
+
+ - name: Shutdown docker compose services
+ if: runner.os == 'Linux'
+ run: |
+ docker compose down
diff --git a/.github/workflows/r-extended.yml b/.github/workflows/r-extended.yml
new file mode 100644
index 00000000..ccfd898a
--- /dev/null
+++ b/.github/workflows/r-extended.yml
@@ -0,0 +1,147 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: R (extended)
+
+# Runs weekly. This catches changes that happen because of
+# dependency updates and catches some of the more infrequent/obscure
+# changes that affect the R package.
+on:
+ # Also can trigger manually (e.g., if release is upcoming, etc.)
+ workflow_dispatch:
+
+ schedule:
+ - cron: '5 0 * * 0'
+
+ pull_request:
+ branches:
+ - main
+ paths:
+ - ".github/workflows/r-extended.yml"
+
+permissions:
+ contents: read
+
+jobs:
+
+ # Runs R CMD check on the same platforms/R versions CRAN does
+ cran:
+ strategy:
+ matrix:
+ rversion: [oldrel, release, devel]
+ os: [macOS, windows, ubuntu]
+ pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql, adbcflightsql,
adbcsnowflake]
+ fail-fast: false
+
+ uses: ./.github/workflows/r-check.yml
+ with:
+ os: ${{ matrix.os }}
+ pkg: ${{ matrix.pkg }}
+ rversion: ${{ matrix.rversion }}
+ secrets:
+ SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
+
+ # Check older versions of R on Linux. This catches accidental use of newer R
functions.
+ rversions:
+ strategy:
+ matrix:
+ rversion: ["3.6", "4.0", "4.1"]
+ os: [ubuntu]
+ pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql, adbcflightsql,
adbcsnowflake]
+ fail-fast: false
+
+ uses: ./.github/workflows/r-check.yml
+ with:
+ os: ${{ matrix.os }}
+ pkg: ${{ matrix.pkg }}
+ rversion: ${{ matrix.rversion }}
+ secrets:
+ SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
+
+ # Checks on older verions of R on Windows. The Windows build system changed
+ # several times so we need to check packages on every version. Go-based
+ # drivers aren't supported before 4.2, so we don't check them here.
+ # We don't need to check R 4.1 because the build system for R 4.0 and R 4.1
+ # are the same.
+ winrversions:
+ strategy:
+ matrix:
+ rversion: ["3.6", "4.0"]
+ os: [windows]
+ pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql]
+ fail-fast: false
+
+ uses: ./.github/workflows/r-check.yml
+ with:
+ os: ${{ matrix.os }}
+ pkg: ${{ matrix.pkg }}
+ rversion: ${{ matrix.rversion }}
+ secrets:
+ SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
+
+ # Runs tests with valgrind. Go does not support valgrind, so we don't run
+ # those tests here.
+ valgrind:
+ name: "valgrind (${{ matrix.pkg }})"
+ runs-on: ${{ matrix.os }}-latest
+
+ strategy:
+ matrix:
+ rversion: [release]
+ os: [ubuntu]
+ pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql]
+ fail-fast: false
+
+ steps:
+ - uses: actions/checkout@v4
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ rversion: ${{ matrix.rversion }}
+ use-public-rspm: true
+
+ - name: Install valgrind
+ run: |
+ sudo apt-get install -y valgrind
+
+ - uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ extra-packages: local::../adbcdrivermanager
+ working-directory: r/${{ matrix.pkg }}
+
+ - name: Start postgres test database
+ if: matrix.pkg == 'adbcpostgresql'
+ run: |
+ cd r/adbcpostgresql
+ docker compose up --detach postgres-test
+
ADBC_POSTGRESQL_TEST_URI="postgresql://localhost:5432/postgres?user=postgres&password=password"
+ echo "ADBC_POSTGRESQL_TEST_URI=${ADBC_POSTGRESQL_TEST_URI}" >>
$GITHUB_ENV
+
+ - name: Run devtools::test() with valgrind
+ run: |
+ cd r/${{ matrix.pkg }}
+ R -d "valgrind --tool=memcheck --leak-check=full
--suppressions=../valgrind.supp --error-exitcode=1" -e "testthat::test_local()"
> valgrind.out 2>&1
+
+ - name: Show output
+ if: always()
+ run: |
+ cd r/${{ matrix.pkg }}
+ cat valgrind.out
+
+ - name: Shutdown docker compose services
+ if: always()
+ run: |
+ docker compose down
diff --git a/.github/workflows/r-standard.yml b/.github/workflows/r-standard.yml
new file mode 100644
index 00000000..09cd1f10
--- /dev/null
+++ b/.github/workflows/r-standard.yml
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: R (standard)
+
+# Runs on PRs that touch the R packages and when pushing files the R
+# package uses to main.
+on:
+ pull_request:
+ branches:
+ - main
+ paths:
+ - "r/**"
+ - ".github/workflows/r-standard.yml"
+ - ".github/workflows/r-check.yml"
+
+ push:
+ branches:
+ - main
+ paths:
+ - "adbc.h"
+ - "c/**"
+ - "go/adbc/driver/**"
+ - "go/adbc/pkg/**"
+ - "r/**"
+ - ".github/workflows/r-basic.yml"
+ - ".github/workflows/r-check.yml"
+
+concurrency:
+ group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ check:
+ strategy:
+ matrix:
+ os: [ubuntu, macOS, windows]
+ pkg: [adbcdrivermanager, adbcsqlite, adbcpostgresql, adbcflightsql,
adbcsnowflake]
+
+ uses: ./.github/workflows/r-check.yml
+ with:
+ os: ${{ matrix.os }}
+ pkg: ${{ matrix.pkg }}
+ secrets:
+ SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
diff --git a/r/valgrind.supp b/r/valgrind.supp
new file mode 100644
index 00000000..01ee4bce
--- /dev/null
+++ b/r/valgrind.supp
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{
+ <cgo>:Thread locals don't appear to be freed
+ Memcheck:Leak
+ ...
+ fun:_dl_allocate_tls
+}
