This is an automated email from the ASF dual-hosted git repository.
yikun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-docker.git
The following commit(s) were added to refs/heads/master by this push:
new d58e178 [SPARK-41269][INFRA] Move image matrix into version's workflow
d58e178 is described below
commit d58e17890f07b4c8c8d212775a53c48dc3a6ce42
Author: Yikun Jiang <yikunk...@gmail.com>
AuthorDate: Mon Nov 28 09:36:54 2022 +0800
[SPARK-41269][INFRA] Move image matrix into version's workflow
### What changes were proposed in this pull request?
This patch refactors main workflow:
- Move image matrix into version's workflow to make the main workflow more
clear. And also will help downstream repo to only validate specified image type.
- Move build steps into a same section
### Why are the changes needed?
This will help downstream repo to only validate specified image type.
After this patch, we will add a test to reuse spark docker workflow like:
https://github.com/yikun/spark-docker/commit/45044cee2e8919de7e7353e74f8ca612ad16629a
to help developers/users test their self build image.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
CI passed
Closes #25 from Yikun/matrix-refactor.
Authored-by: Yikun Jiang <yikunk...@gmail.com>
Signed-off-by: Yikun Jiang <yikunk...@gmail.com>
---
.github/workflows/build_3.3.0.yaml | 4 ++
.github/workflows/build_3.3.1.yaml | 4 ++
.github/workflows/main.yml | 76 ++++++++++++++++++++------------------
.github/workflows/publish.yml | 2 +
4 files changed, 51 insertions(+), 35 deletions(-)
diff --git a/.github/workflows/build_3.3.0.yaml
b/.github/workflows/build_3.3.0.yaml
index 7e7ce39..a4f8224 100644
--- a/.github/workflows/build_3.3.0.yaml
+++ b/.github/workflows/build_3.3.0.yaml
@@ -30,6 +30,9 @@ on:
jobs:
run-build:
+ strategy:
+ matrix:
+ image-type: ["all", "python", "scala", "r"]
name: Run
secrets: inherit
uses: ./.github/workflows/main.yml
@@ -37,3 +40,4 @@ jobs:
spark: 3.3.0
scala: 2.12
java: 11
+ image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/build_3.3.1.yaml
b/.github/workflows/build_3.3.1.yaml
index f6a4b7d..9e5c082 100644
--- a/.github/workflows/build_3.3.1.yaml
+++ b/.github/workflows/build_3.3.1.yaml
@@ -30,6 +30,9 @@ on:
jobs:
run-build:
+ strategy:
+ matrix:
+ image-type: ["all", "python", "scala", "r"]
name: Run
secrets: inherit
uses: ./.github/workflows/main.yml
@@ -37,3 +40,4 @@ jobs:
spark: 3.3.1
scala: 2.12
java: 11
+ image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 024b853..ebafcdc 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -47,6 +47,11 @@ on:
required: false
type: string
default: ghcr.io/apache/spark-docker
+ image-type:
+ description: The image type of the image (all, python, scala, r).
+ required: false
+ type: string
+ default: python
jobs:
main:
@@ -60,41 +65,33 @@ jobs:
image: registry:2
ports:
- 5000:5000
- strategy:
- matrix:
- spark_version:
- - ${{ inputs.spark }}
- scala_version:
- - ${{ inputs.scala }}
- java_version:
- - ${{ inputs.java }}
- image_suffix: [python3-ubuntu, ubuntu, r-ubuntu, python3-r-ubuntu]
steps:
- name: Checkout Spark Docker repository
uses: actions/checkout@v3
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v2
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
- with:
- # This required by local registry
- driver-opts: network=host
-
- - name: Generate tags
+ - name: Prepare - Generate tags
run: |
- TAG=scala${{ matrix.scala_version }}-java${{ matrix.java_version
}}-${{ matrix.image_suffix }}
+ case "${{ inputs.image-type }}" in
+ all) SUFFIX=python3-r-ubuntu
+ ;;
+ python) SUFFIX=python3-ubuntu
+ ;;
+ r) SUFFIX=r-ubuntu
+ ;;
+ scala) SUFFIX=ubuntu
+ ;;
+ esac
+ TAG=scala${{ inputs.scala }}-java${{ inputs.java }}-$SUFFIX
REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]'
'[:lower:]')
TEST_REPO=localhost:5000/$REPO_OWNER/spark-docker
IMAGE_NAME=spark
- IMAGE_PATH=${{ matrix.spark_version }}/$TAG
- UNIQUE_IMAGE_TAG=${{ matrix.spark_version }}-$TAG
+ IMAGE_PATH=${{ inputs.spark }}/$TAG
+ UNIQUE_IMAGE_TAG=${{ inputs.spark }}-$TAG
IMAGE_URL=$TEST_REPO/$IMAGE_NAME:$UNIQUE_IMAGE_TAG
PUBLISH_REPO=${{ inputs.repository }}
- PUBLISH_IMAGE_URL=`tools/manifest.py tags -i
${PUBLISH_REPO}/${IMAGE_NAME} -p ${{ matrix.spark_version }}/${TAG}`
+ PUBLISH_IMAGE_URL=`tools/manifest.py tags -i
${PUBLISH_REPO}/${IMAGE_NAME} -p ${{ inputs.spark }}/${TAG}`
# Unique image tag in each version:
3.3.0-scala2.12-java11-python3-ubuntu
echo "UNIQUE_IMAGE_TAG=${UNIQUE_IMAGE_TAG}" >> $GITHUB_ENV
@@ -110,7 +107,7 @@ jobs:
echo "PUBLISH_REPO=${PUBLISH_REPO}" >> $GITHUB_ENV
echo "PUBLISH_IMAGE_URL=${PUBLISH_IMAGE_URL}" >> $GITHUB_ENV
- - name: Print Image tags
+ - name: Prepare - Print Image tags
run: |
echo "UNIQUE_IMAGE_TAG: "${UNIQUE_IMAGE_TAG}
echo "TEST_REPO: "${TEST_REPO}
@@ -121,7 +118,16 @@ jobs:
echo "PUBLISH_REPO:"${PUBLISH_REPO}
echo "PUBLISH_IMAGE_URL:"${PUBLISH_IMAGE_URL}
- - name: Build and push test image
+ - name: Build - Set up QEMU
+ uses: docker/setup-qemu-action@v2
+
+ - name: Build - Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+ with:
+ # This required by local registry
+ driver-opts: network=host
+
+ - name: Build - Build and push test image
uses: docker/build-push-action@v3
with:
context: ${{ env.IMAGE_PATH }}
@@ -130,20 +136,20 @@ jobs:
push: true
- name : Test - Run spark application for standalone cluster on docker
- run: testing/run_tests.sh --image-url $IMAGE_URL --scala-version ${{
matrix.scala_version }} --spark-version ${{ matrix.spark_version }}
+ run: testing/run_tests.sh --image-url $IMAGE_URL --scala-version ${{
inputs.scala }} --spark-version ${{ inputs.spark }}
- name: Test - Checkout Spark repository
uses: actions/checkout@v3
with:
fetch-depth: 0
repository: apache/spark
- ref: v${{ matrix.spark_version }}
+ ref: v${{ inputs.spark }}
path: ${{ github.workspace }}/spark
- name: Test - Cherry pick commits
# Apache Spark enable resource limited k8s IT since v3.3.1,
cherry-pick patches for old release
# https://github.com/apache/spark/pull/36087#issuecomment-1251756266
- if: matrix.spark_version == '3.3.0'
+ if: inputs.spark == '3.3.0'
working-directory: ${{ github.workspace }}/spark
run: |
# SPARK-38802: Add driverRequestCores/executorRequestCores supported
@@ -159,7 +165,7 @@ jobs:
# This is required after v2, now just keep same distribution with v1
# https://github.com/actions/setup-java/releases/tag/v2.0.0
distribution: 'zulu'
- java-version: ${{ matrix.java_version }}
+ java-version: ${{ inputs.java }}
- name: Test - Cache Scala, SBT and Maven
uses: actions/cache@v3
@@ -169,13 +175,13 @@ jobs:
build/scala-*
build/*.jar
~/.sbt
- key: build-${{ matrix.spark_version }}-scala${{ matrix.scala_version
}}-java${{ matrix.java_version }}
+ key: build-${{ inputs.spark }}-scala${{ inputs.scala }}-java${{
inputs.java }}
- name: Test - Cache Coursier local repository
uses: actions/cache@v3
with:
path: ~/.cache/coursier
- key: build-${{ matrix.spark_version }}-scala${{ matrix.scala_version
}}-java${{ matrix.java_version }}-coursier
+ key: build-${{ inputs.spark }}-scala${{ inputs.scala }}-java${{
inputs.java }}-coursier
- name: Test - Start minikube
run: |
@@ -205,11 +211,11 @@ jobs:
OPTS+="-Dspark.kubernetes.test.pythonImage=${IMAGE_NAME} "
OPTS+="-Dspark.kubernetes.test.rImage=${IMAGE_NAME} "
- if echo ${{ matrix.image_suffix }} | grep -q "python3-r-ubuntu"; then
+ if [ "${{ inputs.image-type }}" = "all" ]; then
# Prepare test jar for client tests
CONTAINER_TMP_NAME=spark-example-image
docker create -ti --name $CONTAINER_TMP_NAME ${{ env.IMAGE_URL }}
bash
- docker cp
$CONTAINER_TMP_NAME:/opt/spark/examples/jars/spark-examples_${{
matrix.scala_version }}-${{ matrix.spark_version }}.jar .
+ docker cp
$CONTAINER_TMP_NAME:/opt/spark/examples/jars/spark-examples_${{ inputs.scala
}}-${{ inputs.spark }}.jar .
docker rm -f $CONTAINER_TMP_NAME
# Prepare PV test
PVC_TMP_DIR=$(mktemp -d)
@@ -223,12 +229,12 @@ jobs:
build/sbt $OPTS 'kubernetes-integration-tests/testOnly -- -z "Run
SparkPi"'
# Run basic test for PySpark image
- if echo ${{ matrix.image_suffix }} | grep -q "python"; then
+ if [ "${{ inputs.image-type }}" = "python" ]; then
build/sbt $OPTS 'kubernetes-integration-tests/testOnly -- -z
"Run PySpark"'
fi
# Run basic test for SparkR image
- if echo ${{ matrix.image_suffix }} | grep -q "r-"; then
+ if [ "${{ inputs.image-type }}" = "r" ]; then
OPTS+="-Psparkr -Dtest.include.tags=r "
build/sbt $OPTS 'kubernetes-integration-tests/testOnly'
fi
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index a44153b..4a07f5d 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -53,6 +53,7 @@ jobs:
matrix:
scala: [2.12]
java: [11]
+ image-type: ["all", "python", "scala", "r"]
permissions:
packages: write
name: Run
@@ -64,3 +65,4 @@ jobs:
java: ${{ matrix.java }}
publish: ${{ inputs.publish }}
repository: ${{ inputs.repository }}
+ image-type: ${{ matrix.image-type }}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
