This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 1928257ce3 Build: Merge job definitions in spark-ci.yml (#10513)
1928257ce3 is described below
commit 1928257ce3d8ec6dfa96fb9d8424294b27bc3f72
Author: Piotr Findeisen <[email protected]>
AuthorDate: Mon Jun 17 15:11:11 2024 +0200
Build: Merge job definitions in spark-ci.yml (#10513)
Refactor to remove code duplication. Does not change test coverage nor
configurations exercised on CI.
---
.github/workflows/spark-ci.yml | 78 +++++-------------------------------------
1 file changed, 9 insertions(+), 69 deletions(-)
diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml
index ef028fcd76..5ddff7a506 100644
--- a/.github/workflows/spark-ci.yml
+++ b/.github/workflows/spark-ci.yml
@@ -65,12 +65,13 @@ concurrency:
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
- spark-3x-scala-2-12-tests:
+ spark-tests:
runs-on: ubuntu-22.04
strategy:
matrix:
- jvm: [8, 11]
+ jvm: [8, 11, 17]
spark: ['3.3', '3.4', '3.5']
+ scala: ['2.12', '2.13']
env:
SPARK_LOCAL_IP: localhost
steps:
@@ -90,73 +91,12 @@ jobs:
with:
tool-cache: false
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' |
cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.12
-DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark
}}_2.12:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark
}}_2.12:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark
}}_2.12:check -Pquick=true -x javadoc
- - uses: actions/upload-artifact@v4
- if: failure()
- with:
- name: test logs
- path: |
- **/build/testlogs
-
- spark-3x-scala-2-13-tests:
- runs-on: ubuntu-22.04
- strategy:
- matrix:
- jvm: [8, 11]
- spark: ['3.3','3.4','3.5']
- env:
- SPARK_LOCAL_IP: localhost
- steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-java@v4
- with:
- distribution: zulu
- java-version: ${{ matrix.jvm }}
- - uses: actions/cache@v4
- with:
- path: |
- ~/.gradle/caches
- ~/.gradle/wrapper
- key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*',
'**/gradle-wrapper.properties') }}
- restore-keys: ${{ runner.os }}-gradle-
- - uses:
jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
- with:
- tool-cache: false
- - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' |
cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.13
-DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark
}}_2.13:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark
}}_2.13:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark
}}_2.13:check -Pquick=true -x javadoc
- - uses: actions/upload-artifact@v4
- if: failure()
- with:
- name: test logs
- path: |
- **/build/testlogs
-
- spark-3x-java-17-tests:
- runs-on: ubuntu-22.04
- strategy:
- matrix:
- spark: ['3.3','3.4', '3.5']
- scala-version: ['2.12', '2.13']
- env:
- SPARK_LOCAL_IP: localhost
- steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-java@v4
- with:
- distribution: zulu
- java-version: 17
- - uses: actions/cache@v4
- with:
- path: |
- ~/.gradle/caches
- ~/.gradle/wrapper
- key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*',
'**/gradle-wrapper.properties') }}
- restore-keys: ${{ runner.os }}-gradle-
- - uses:
jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
- with:
- tool-cache: false
- - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' |
cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=${{
matrix.scala-version }} -DhiveVersions= -DflinkVersions=
:iceberg-spark:iceberg-spark-${{ matrix.spark }}_${{ matrix.scala-version
}}:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_${{
matrix.scala-version }}:check :iceberg-spark:iceberg-spark-runtime-${{
matrix.spark }}_${{ matrix.scala-version }}:check -Pquick=true -x javadoc
+ - run: |
+ ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=${{
matrix.scala }} -DhiveVersions= -DflinkVersions= \
+ :iceberg-spark:iceberg-spark-${{ matrix.spark }}_${{ matrix.scala
}}:check \
+ :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_${{
matrix.scala }}:check \
+ :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_${{
matrix.scala }}:check \
+ -Pquick=true -x javadoc
- uses: actions/upload-artifact@v4
if: failure()
with:
