This is an automated email from the ASF dual-hosted git repository.
yikun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-docker.git
The following commit(s) were added to refs/heads/master by this push:
new f2d2b2d [SPARK-43365][FOLLWUP] Refactor publish workflow based on
base image
f2d2b2d is described below
commit f2d2b2d1ffbb951aed29221a040861327c09441e
Author: Yikun Jiang <yikunk...@gmail.com>
AuthorDate: Thu May 25 16:13:44 2023 +0800
[SPARK-43365][FOLLWUP] Refactor publish workflow based on base image
### What changes were proposed in this pull request?
- This patch changes the `build-args` to `patch in test` in build and
publish workflow, because the docker official image do not support
**parameterized FROM** values.
https://github.com/docker-library/official-images/pull/13089#issuecomment-1555352902
- And also Refactor publish workflow:
### Why are the changes needed?
Same change with build workflow refactor, to avoid the publish issue like:
```
#5 [linux/amd64 internal] load metadata for
docker.io/library/spark:3.4.0-scala2.12-java11-ubuntu
#5 ERROR: pull access denied, repository does not exist or may require
authorization: server message: insufficient_scope: authorization failed
------
> [linux/amd64 internal] load metadata for
docker.io/library/spark:3.4.0-scala2.12-java11-ubuntu:
------
Dockerfile:18
--------------------
16 | #
17 | ARG BASE_IMAGE=spark:3.4.0-scala2.12-java11-ubuntu
18 | >>> FROM $BASE_IMAGE
19 |
20 | RUN set -ex && \
--------------------
ERROR: failed to solve: spark:3.4.0-scala2.12-java11-ubuntu: pull access
denied, repository does not exist or may require authorization: server message:
insufficient_scope: authorization failed
Error: buildx failed with: ERROR: failed to solve:
spark:3.4.0-scala2.12-java11-ubuntu: pull access denied, repository does not
exist or may require authorization: server message: insufficient_scope:
authorization failed
```
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Publish test in my local fork:
-
https://github.com/Yikun/spark-docker/actions/runs/5076986823/jobs/9120029759:
Skip the local base build use the [published
base](https://github.com/Yikun/spark-docker/actions/runs/5076986823/jobs/9120029759#step:11:135)
image:
```
#3 [linux/amd64 internal] load metadata for
ghcr.io/yikun/spark-docker/spark:3.4.0-scala2.12-java11-ubuntu
#3 DONE 0.9s
#4 [linux/arm64 internal] load metadata for
ghcr.io/yikun/spark-docker/spark:3.4.0-scala2.12-java11-ubuntu
#4 DONE 0.9s
```
- CI passed: do local base build first and build base on the local build
Closes #39 from Yikun/publish-build.
Authored-by: Yikun Jiang <yikunk...@gmail.com>
Signed-off-by: Yikun Jiang <yikunk...@gmail.com>
---
.github/workflows/main.yml | 21 ++++++++++++++++--
.github/workflows/publish.yml | 25 +++++++++++++++++++++-
3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile | 3 +--
3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile | 3 +--
3.4.0/scala2.12-java11-r-ubuntu/Dockerfile | 3 +--
r-python.template | 3 +--
6 files changed, 47 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c1d0c56..870c8c7 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -107,6 +107,9 @@ jobs:
TEST_REPO=${{ inputs.repository }}
UNIQUE_IMAGE_TAG=${{ inputs.image-tag }}
fi
+
+ # We can't use the real image for build because we haven't publish
the image yet.
+ # The base image for build, it's something like
localhost:5000/$REPO_OWNER/spark-docker/spark:3.3.0-scala2.12-java11-ubuntu
BASE_IMAGE_URL=$TEST_REPO/$IMAGE_NAME:$BASE_IMGAE_TAG
IMAGE_URL=$TEST_REPO/$IMAGE_NAME:$UNIQUE_IMAGE_TAG
@@ -157,7 +160,8 @@ jobs:
driver-opts: network=host
- name: Build - Build the base image
- if: ${{ inputs.build }}
+ # Don't need to build the base image when publish
+ if: ${{ inputs.build && !inputs.publish }}
uses: docker/build-push-action@v3
with:
context: ${{ env.BASE_IMAGE_PATH }}
@@ -165,11 +169,24 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
+ - name: Build - Use the test image repo when build
+ # Don't need to build the base image when publish
+ if: ${{ inputs.build && !inputs.publish }}
+ working-directory: ${{ env.IMAGE_PATH }}
+ run: |
+ sed -i "s@FROM spark@FROM $TEST_REPO/spark@g" ./Dockerfile
+
+ - name: Build - Use real image repo when publish
+ # Don't need to build the base image when publish
+ if: ${{ inputs.publish }}
+ working-directory: ${{ env.IMAGE_PATH }}
+ run: |
+ sed -i "s@FROM spark@FROM $PUBLISH_REPO/spark@g" ./Dockerfile
+
- name: Build - Build and push test image
if: ${{ inputs.build }}
uses: docker/build-push-action@v3
with:
- build-args: BASE_IMAGE=${{ env.BASE_IMAGE_URL }}
context: ${{ env.IMAGE_PATH }}
tags: ${{ env.IMAGE_URL }}
platforms: linux/amd64,linux/arm64
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 70b88b8..3063bfe 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -49,13 +49,36 @@ on:
- apache
jobs:
+ # We first build and publish the base image
+ run-base-build:
+ # if: startsWith(inputs.spark, '3.3')
+ strategy:
+ matrix:
+ scala: [2.12]
+ java: [11]
+ image-type: ["scala"]
+ permissions:
+ packages: write
+ name: Run Base
+ secrets: inherit
+ uses: ./.github/workflows/main.yml
+ with:
+ spark: ${{ inputs.spark }}
+ scala: ${{ matrix.scala }}
+ java: ${{ matrix.java }}
+ publish: ${{ inputs.publish }}
+ repository: ${{ inputs.repository }}
+ image-type: ${{ matrix.image-type }}
+
+ # Then publish the all / python / r images
run-build:
+ needs: run-base-build
# if: startsWith(inputs.spark, '3.3')
strategy:
matrix:
scala: [2.12]
java: [11]
- image-type: ["all", "python", "scala", "r"]
+ image-type: ["all", "python", "r"]
permissions:
packages: write
name: Run
diff --git a/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
b/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
index 12c7a4f..7734100 100644
--- a/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ b/3.4.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-ARG BASE_IMAGE=spark:3.4.0-scala2.12-java11-ubuntu
-FROM $BASE_IMAGE
+FROM spark:3.4.0-scala2.12-java11-ubuntu
RUN set -ex; \
apt-get update; \
diff --git a/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile
b/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile
index 1f0dd1f..6c12c30 100644
--- a/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile
+++ b/3.4.0/scala2.12-java11-python3-ubuntu/Dockerfile
@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-ARG BASE_IMAGE=spark:3.4.0-scala2.12-java11-ubuntu
-FROM $BASE_IMAGE
+FROM spark:3.4.0-scala2.12-java11-ubuntu
RUN set -ex; \
apt-get update; \
diff --git a/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile
b/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile
index 53647b2..24cd41a 100644
--- a/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile
+++ b/3.4.0/scala2.12-java11-r-ubuntu/Dockerfile
@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-ARG BASE_IMAGE=spark:3.4.0-scala2.12-java11-ubuntu
-FROM $BASE_IMAGE
+FROM spark:3.4.0-scala2.12-java11-ubuntu
RUN set -ex; \
apt-get update; \
diff --git a/r-python.template b/r-python.template
index 1c3087e..d3f4ef7 100644
--- a/r-python.template
+++ b/r-python.template
@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-ARG BASE_IMAGE=spark:{{ SPARK_VERSION }}-scala{{ SCALA_VERSION }}-java{{
JAVA_VERSION }}-ubuntu
-FROM $BASE_IMAGE
+FROM spark:{{ SPARK_VERSION }}-scala{{ SCALA_VERSION }}-java{{ JAVA_VERSION
}}-ubuntu
RUN set -ex; \
apt-get update; \
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
