This is an automated email from the ASF dual-hosted git repository. pingsutw pushed a commit to branch SUBMARINE-1393 in repository https://gitbox.apache.org/repos/asf/submarine.git
commit 850770e47384818922e60da1d88efc6d7b1585c7 Author: Kevin Su <[email protected]> AuthorDate: Sat Sep 23 00:57:48 2023 -0700 Use buildx to build images Signed-off-by: Kevin Su <[email protected]> --- .github/workflows/deploy_docker_images.yml | 40 ++++++---------------- dev-support/cicd/build.sh | 2 +- dev-support/docker-images/agent/build.sh | 2 +- dev-support/docker-images/database/build.sh | 2 +- .../docker-images/experiment-prehandler/build.sh | 4 +-- dev-support/docker-images/jupyter-gpu/build.sh | 2 +- dev-support/docker-images/jupyter/build.sh | 2 +- dev-support/docker-images/mlflow/build.sh | 2 +- dev-support/docker-images/operator-v3/build.sh | 2 +- dev-support/docker-images/operator/build.sh | 2 +- dev-support/docker-images/serve/build.sh | 2 +- dev-support/docker-images/submarine/build.sh | 2 +- dev-support/examples/mnist-pytorch/DDP/build.sh | 2 +- .../mnist-tensorflow/MirroredStrategy/build.sh | 2 +- .../MultiWorkerMirroredStrategy/build.sh | 2 +- .../ParameterServerStrategy/build.sh | 2 +- dev-support/examples/nn-pytorch/build.sh | 2 +- dev-support/examples/quickstart/build.sh | 2 +- dev-support/examples/tracking/build.sh | 2 +- dev-support/misc/flask/build.sh | 2 +- 20 files changed, 30 insertions(+), 50 deletions(-) diff --git a/.github/workflows/deploy_docker_images.yml b/.github/workflows/deploy_docker_images.yml index 849f49163..d529ff90d 100644 --- a/.github/workflows/deploy_docker_images.yml +++ b/.github/workflows/deploy_docker_images.yml @@ -50,52 +50,32 @@ jobs: - name: Build with Maven run: mvn clean install -DskipTests - - name: Build submarine server + - name: Build and push submarine server run: ./dev-support/docker-images/submarine/build.sh - - name: Push submarine-server docker image - run: docker push apache/submarine:server-$SUBMARINE_VERSION - - name: Build submarine agent + - name: Build and push submarine agent run: ./dev-support/docker-images/agent/build.sh - - name: Push submarine-agent docker image - run: docker push apache/submarine:agent-$SUBMARINE_VERSION - - name: Build submarine database + - name: Build and push submarine database run: ./dev-support/docker-images/database/build.sh - - name: Push submarine-database docker image - run: docker push apache/submarine:database-$SUBMARINE_VERSION - - name: Build submarine jupyter + - name: Build and push submarine jupyter run: ./dev-support/docker-images/jupyter/build.sh - - name: Push submarine-jupyter docker image - run: docker push apache/submarine:jupyter-notebook-$SUBMARINE_VERSION - - name: Build submarine jupyter gpu + - name: Build and push submarine jupyter gpu run: ./dev-support/docker-images/jupyter-gpu/build.sh - - name: Push submarine-jupyter-gpu docker image - run: docker push apache/submarine:jupyter-notebook-gpu-$SUBMARINE_VERSION - - name: Build submarine operator + - name: Build and push submarine operator run: ./dev-support/docker-images/operator-v3/build.sh - - name: Push submarine-operator docker image - run: docker push apache/submarine:operator-$SUBMARINE_VERSION - - name: Build submarine mlflow + - name: Build and push submarine mlflow run: ./dev-support/docker-images/mlflow/build.sh - - name: Push submarine-mlflow docker image - run: docker push apache/submarine:mlflow-$SUBMARINE_VERSION - - name: Build submarine serve + - name: Build and push submarine serve run: ./dev-support/docker-images/serve/build.sh - - name: Push submarine-serve docker image - run: docker push apache/submarine:serve-$SUBMARINE_VERSION - - name: Build submarine quickstart + - name: Build and push submarine quickstart run: ./dev-support/examples/quickstart/build.sh - - name: Push submarine quickstart docker image - run: docker push apache/submarine:quickstart-$SUBMARINE_VERSION - - name: Build submarine cicd + - name: Build and push submarine cicd run: ./dev-support/cicd/build.sh - - name: Push submarine-cicd docker image - run: docker push apache/submarine:cicd-$SUBMARINE_VERSION diff --git a/dev-support/cicd/build.sh b/dev-support/cicd/build.sh index 3fb7e4bcc..2afd37e87 100755 --- a/dev-support/cicd/build.sh +++ b/dev-support/cicd/build.sh @@ -29,4 +29,4 @@ SUBMARINE_HOME=${CURRENT_PATH}/../../.. # build image echo "Start building the ${IMAGE} docker image ..." cd ${CURRENT_PATH} -docker build -t ${IMAGE} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${IMAGE} . diff --git a/dev-support/docker-images/agent/build.sh b/dev-support/docker-images/agent/build.sh index 5330f6378..5e8639eb3 100755 --- a/dev-support/docker-images/agent/build.sh +++ b/dev-support/docker-images/agent/build.sh @@ -44,7 +44,7 @@ cp ${SUBMARINE_HOME}/submarine-server/server-submitter/submarine-k8s-agent/targe cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/docker-images/database/build.sh b/dev-support/docker-images/database/build.sh index 1cae44770..d0bf35ced 100755 --- a/dev-support/docker-images/database/build.sh +++ b/dev-support/docker-images/database/build.sh @@ -32,7 +32,7 @@ cp -rf "${SUBMARINE_HOME}/dev-support/database" "${CURRENT_PATH}" # build image echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." cd ${CURRENT_PATH} -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean template file rm -rf ${CURRENT_PATH}/database diff --git a/dev-support/docker-images/experiment-prehandler/build.sh b/dev-support/docker-images/experiment-prehandler/build.sh index c1e94a379..a3723410a 100755 --- a/dev-support/docker-images/experiment-prehandler/build.sh +++ b/dev-support/docker-images/experiment-prehandler/build.sh @@ -39,12 +39,12 @@ trap "test -f $tmpfile && rm $tmpfile" RETURN curl -L -o $tmpfile ${HADOOP_TAR_URL} mv $tmpfile ${CURRENT_PATH}/tmp/hadoop-3.3.3.tar.gz -curl -L -o ${CURRENT_PATH}/tmp/hadoop-aws-3.3.3.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.3/hadoop-aws-3.3.3.jar +curl -L -o ${CURRENT_PATH}/tmp/hadoop-aws-3.3.3.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.3/hadoop-aws-3.3.3.jar curl -L -o ${CURRENT_PATH}/tmp/aws-java-sdk-bundle-1.12.267.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.267/aws-java-sdk-bundle-1.12.267.jar echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." cd ${CURRENT_PATH} -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/docker-images/jupyter-gpu/build.sh b/dev-support/docker-images/jupyter-gpu/build.sh index 036bb421a..04ea42b06 100755 --- a/dev-support/docker-images/jupyter-gpu/build.sh +++ b/dev-support/docker-images/jupyter-gpu/build.sh @@ -29,4 +29,4 @@ SUBMARINE_HOME=${CURRENT_PATH}/../../.. # build image echo "Start building the ${JUPYTER_IMAGE} docker image ..." cd ${CURRENT_PATH} -docker build -t ${JUPYTER_IMAGE} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${JUPYTER_IMAGE} . diff --git a/dev-support/docker-images/jupyter/build.sh b/dev-support/docker-images/jupyter/build.sh index b1d24a81e..0028cf8c9 100755 --- a/dev-support/docker-images/jupyter/build.sh +++ b/dev-support/docker-images/jupyter/build.sh @@ -29,4 +29,4 @@ SUBMARINE_HOME=${CURRENT_PATH}/../../.. # build image echo "Start building the ${JUPYTER_IMAGE} docker image ..." cd ${CURRENT_PATH} -docker build -t ${JUPYTER_IMAGE} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${JUPYTER_IMAGE} . diff --git a/dev-support/docker-images/mlflow/build.sh b/dev-support/docker-images/mlflow/build.sh index 4a00c5df5..b6d014f93 100755 --- a/dev-support/docker-images/mlflow/build.sh +++ b/dev-support/docker-images/mlflow/build.sh @@ -29,4 +29,4 @@ export SUBMARINE_HOME=${CURRENT_PATH}/../../.. # build image echo "Start building the ${MLFLOW_IMAGE} docker image ..." cd ${CURRENT_PATH} -docker build -t ${MLFLOW_IMAGE} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${MLFLOW_IMAGE} . diff --git a/dev-support/docker-images/operator-v3/build.sh b/dev-support/docker-images/operator-v3/build.sh index eca60d948..3e50eb171 100755 --- a/dev-support/docker-images/operator-v3/build.sh +++ b/dev-support/docker-images/operator-v3/build.sh @@ -32,7 +32,7 @@ cp -r ${SUBMARINE_HOME}/submarine-cloud-v3/ "${CURRENT_PATH}/tmp/submarine-cloud cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/docker-images/operator/build.sh b/dev-support/docker-images/operator/build.sh index 1320526c0..9737c04bf 100755 --- a/dev-support/docker-images/operator/build.sh +++ b/dev-support/docker-images/operator/build.sh @@ -32,7 +32,7 @@ cp -r ${SUBMARINE_HOME}/submarine-cloud-v2/ "${CURRENT_PATH}/tmp/submarine-cloud cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/docker-images/serve/build.sh b/dev-support/docker-images/serve/build.sh index d8cc6a1ed..d6ed9676f 100755 --- a/dev-support/docker-images/serve/build.sh +++ b/dev-support/docker-images/serve/build.sh @@ -30,4 +30,4 @@ export SUBMARINE_HOME=${CURRENT_PATH}/../../.. # build image cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . diff --git a/dev-support/docker-images/submarine/build.sh b/dev-support/docker-images/submarine/build.sh index 56b6720e3..01c450d5a 100755 --- a/dev-support/docker-images/submarine/build.sh +++ b/dev-support/docker-images/submarine/build.sh @@ -59,7 +59,7 @@ cp ${SUBMARINE_HOME}/bin/submarine.sh "${CURRENT_PATH}/tmp/" cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/examples/mnist-pytorch/DDP/build.sh b/dev-support/examples/mnist-pytorch/DDP/build.sh index 6a59a9745..5edfd8580 100755 --- a/dev-support/examples/mnist-pytorch/DDP/build.sh +++ b/dev-support/examples/mnist-pytorch/DDP/build.sh @@ -38,7 +38,7 @@ cp -r "${SUBMARINE_HOME}/submarine-sdk" "${CURRENT_PATH}/tmp" # build image cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/examples/mnist-tensorflow/MirroredStrategy/build.sh b/dev-support/examples/mnist-tensorflow/MirroredStrategy/build.sh index 977c19003..737035169 100755 --- a/dev-support/examples/mnist-tensorflow/MirroredStrategy/build.sh +++ b/dev-support/examples/mnist-tensorflow/MirroredStrategy/build.sh @@ -38,7 +38,7 @@ cp -r "${SUBMARINE_HOME}/submarine-sdk" "${CURRENT_PATH}/tmp" # build image cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/examples/mnist-tensorflow/MultiWorkerMirroredStrategy/build.sh b/dev-support/examples/mnist-tensorflow/MultiWorkerMirroredStrategy/build.sh index bd401bdf3..74985442c 100755 --- a/dev-support/examples/mnist-tensorflow/MultiWorkerMirroredStrategy/build.sh +++ b/dev-support/examples/mnist-tensorflow/MultiWorkerMirroredStrategy/build.sh @@ -38,7 +38,7 @@ cp -r "${SUBMARINE_HOME}/submarine-sdk" "${CURRENT_PATH}/tmp" # build image cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/examples/mnist-tensorflow/ParameterServerStrategy/build.sh b/dev-support/examples/mnist-tensorflow/ParameterServerStrategy/build.sh index c1408937b..a0c937a6f 100755 --- a/dev-support/examples/mnist-tensorflow/ParameterServerStrategy/build.sh +++ b/dev-support/examples/mnist-tensorflow/ParameterServerStrategy/build.sh @@ -38,7 +38,7 @@ cp -r "${SUBMARINE_HOME}/submarine-sdk" "${CURRENT_PATH}/tmp" # build image cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/examples/nn-pytorch/build.sh b/dev-support/examples/nn-pytorch/build.sh index d394f44a7..2b08d8111 100755 --- a/dev-support/examples/nn-pytorch/build.sh +++ b/dev-support/examples/nn-pytorch/build.sh @@ -38,7 +38,7 @@ cp -r "${SUBMARINE_HOME}/submarine-sdk" "${CURRENT_PATH}/tmp" # build image cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/examples/quickstart/build.sh b/dev-support/examples/quickstart/build.sh index cdf99af28..23afe6c71 100755 --- a/dev-support/examples/quickstart/build.sh +++ b/dev-support/examples/quickstart/build.sh @@ -38,7 +38,7 @@ cp -r "${SUBMARINE_HOME}/submarine-sdk" "${CURRENT_PATH}/tmp" # build image cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/examples/tracking/build.sh b/dev-support/examples/tracking/build.sh index 912a2220b..28f7e8f40 100755 --- a/dev-support/examples/tracking/build.sh +++ b/dev-support/examples/tracking/build.sh @@ -38,7 +38,7 @@ cp -r "${SUBMARINE_HOME}/submarine-sdk" "${CURRENT_PATH}/tmp" # build image cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . # clean temp file rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/misc/flask/build.sh b/dev-support/misc/flask/build.sh index 072331283..d21ab54cf 100755 --- a/dev-support/misc/flask/build.sh +++ b/dev-support/misc/flask/build.sh @@ -30,4 +30,4 @@ export SUBMARINE_HOME=${CURRENT_PATH}/../../.. # build image cd ${CURRENT_PATH} echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." -docker build -t ${SUBMARINE_IMAGE_NAME} . +docker buildx build --push --platform linux/amd64,linux/arm64 -t ${SUBMARINE_IMAGE_NAME} . --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
