This is an automated email from the ASF dual-hosted git repository. yikun pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark-docker.git
The following commit(s) were added to refs/heads/master by this push: new 6f68fe0 Add support for java 17 from spark 3.5.0 6f68fe0 is described below commit 6f68fe0f7051c10f2bf43a50a7decfce2e97baf0 Author: vakarisbk <vakaris.bashki...@gmail.com> AuthorDate: Fri Nov 10 11:33:39 2023 +0800 Add support for java 17 from spark 3.5.0 ### What changes were proposed in this pull request? 1. Create Java17 base images alongside Java11 images starting from spark 3.5.0 2. Change ubuntu version to 22.04 for `scala2.12-java17-*` ### Why are the changes needed? Spark supports multiple Java versions, but the images are currently built only with Java 11. ### Does this PR introduce _any_ user-facing change? New images would be available in the repositories. ### How was this patch tested? Closes #56 from vakarisbk/master. Authored-by: vakarisbk <vakaris.bashki...@gmail.com> Signed-off-by: Yikun Jiang <yikunk...@gmail.com> --- .github/workflows/build_3.5.0.yaml | 3 +- .github/workflows/main.yml | 20 +++- .github/workflows/publish.yml | 4 +- .github/workflows/test.yml | 3 + 3.5.0/scala2.12-java17-python3-r-ubuntu/Dockerfile | 29 +++++ 3.5.0/scala2.12-java17-python3-ubuntu/Dockerfile | 26 +++++ 3.5.0/scala2.12-java17-r-ubuntu/Dockerfile | 28 +++++ 3.5.0/scala2.12-java17-ubuntu/Dockerfile | 79 +++++++++++++ 3.5.0/scala2.12-java17-ubuntu/entrypoint.sh | 130 +++++++++++++++++++++ add-dockerfiles.sh | 23 +++- tools/ci_runner_cleaner/free_disk_space.sh | 53 +++++++++ .../ci_runner_cleaner/free_disk_space_container.sh | 33 ++++++ tools/template.py | 2 +- versions.json | 29 +++++ 14 files changed, 454 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_3.5.0.yaml b/.github/workflows/build_3.5.0.yaml index 6eb3ad6..9f2b2d6 100644 --- a/.github/workflows/build_3.5.0.yaml +++ b/.github/workflows/build_3.5.0.yaml @@ -31,11 +31,12 @@ jobs: strategy: matrix: image-type: ["all", "python", "scala", "r"] + java: [11, 17] name: Run secrets: inherit uses: ./.github/workflows/main.yml with: spark: 3.5.0 scala: 2.12 - java: 11 + java: ${{ matrix.java }} image-type: ${{ matrix.image-type }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fe755ed..145b529 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,6 +79,14 @@ jobs: - name: Checkout Spark Docker repository uses: actions/checkout@v3 + - name: Free up disk space + shell: 'script -q -e -c "bash {0}"' + run: | + chmod +x tools/ci_runner_cleaner/free_disk_space_container.sh + tools/ci_runner_cleaner/free_disk_space_container.sh + chmod +x tools/ci_runner_cleaner/free_disk_space.sh + tools/ci_runner_cleaner/free_disk_space.sh + - name: Prepare - Generate tags run: | case "${{ inputs.image-type }}" in @@ -195,7 +203,8 @@ jobs: - name : Test - Run spark application for standalone cluster on docker run: testing/run_tests.sh --image-url $IMAGE_URL --scala-version ${{ inputs.scala }} --spark-version ${{ inputs.spark }} - - name: Test - Checkout Spark repository + - name: Test - Checkout Spark repository for Spark 3.3.0 (with fetch-depth 0) + if: inputs.spark == '3.3.0' uses: actions/checkout@v3 with: fetch-depth: 0 @@ -203,6 +212,14 @@ jobs: ref: v${{ inputs.spark }} path: ${{ github.workspace }}/spark + - name: Test - Checkout Spark repository + if: inputs.spark != '3.3.0' + uses: actions/checkout@v3 + with: + repository: apache/spark + ref: v${{ inputs.spark }} + path: ${{ github.workspace }}/spark + - name: Test - Cherry pick commits # Apache Spark enable resource limited k8s IT since v3.3.1, cherry-pick patches for old release # https://github.com/apache/spark/pull/36087#issuecomment-1251756266 @@ -247,6 +264,7 @@ jobs: # TODO(SPARK-44495): Resume to use the latest minikube for k8s-integration-tests. curl -LO https://storage.googleapis.com/minikube/releases/v1.30.1/minikube-linux-amd64 sudo install minikube-linux-amd64 /usr/local/bin/minikube + rm minikube-linux-amd64 # Github Action limit cpu:2, memory: 6947MB, limit to 2U6G for better resource statistic minikube start --cpus 2 --memory 6144 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8cfa95d..879a9c2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -58,7 +58,7 @@ jobs: strategy: matrix: scala: [2.12] - java: [11] + java: [11, 17] image-type: ["scala"] permissions: packages: write @@ -80,7 +80,7 @@ jobs: strategy: matrix: scala: [2.12] - java: [11] + java: [11, 17] image-type: ["all", "python", "r"] permissions: packages: write diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 47dac20..689981a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,6 +40,9 @@ on: default: 11 type: string required: true + options: + - 11 + - 17 scala: description: 'The Scala version of Spark image.' default: 2.12 diff --git a/3.5.0/scala2.12-java17-python3-r-ubuntu/Dockerfile b/3.5.0/scala2.12-java17-python3-r-ubuntu/Dockerfile new file mode 100644 index 0000000..fd05497 --- /dev/null +++ b/3.5.0/scala2.12-java17-python3-r-ubuntu/Dockerfile @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM spark:3.5.0-scala2.12-java17-ubuntu + +USER root + +RUN set -ex; \ + apt-get update; \ + apt-get install -y python3 python3-pip; \ + apt-get install -y r-base r-base-dev; \ + rm -rf /var/lib/apt/lists/* + +ENV R_HOME /usr/lib/R + +USER spark diff --git a/3.5.0/scala2.12-java17-python3-ubuntu/Dockerfile b/3.5.0/scala2.12-java17-python3-ubuntu/Dockerfile new file mode 100644 index 0000000..e57b4f5 --- /dev/null +++ b/3.5.0/scala2.12-java17-python3-ubuntu/Dockerfile @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM spark:3.5.0-scala2.12-java17-ubuntu + +USER root + +RUN set -ex; \ + apt-get update; \ + apt-get install -y python3 python3-pip; \ + rm -rf /var/lib/apt/lists/* + +USER spark diff --git a/3.5.0/scala2.12-java17-r-ubuntu/Dockerfile b/3.5.0/scala2.12-java17-r-ubuntu/Dockerfile new file mode 100644 index 0000000..753f3cf --- /dev/null +++ b/3.5.0/scala2.12-java17-r-ubuntu/Dockerfile @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM spark:3.5.0-scala2.12-java17-ubuntu + +USER root + +RUN set -ex; \ + apt-get update; \ + apt-get install -y r-base r-base-dev; \ + rm -rf /var/lib/apt/lists/* + +ENV R_HOME /usr/lib/R + +USER spark diff --git a/3.5.0/scala2.12-java17-ubuntu/Dockerfile b/3.5.0/scala2.12-java17-ubuntu/Dockerfile new file mode 100644 index 0000000..ed29cba --- /dev/null +++ b/3.5.0/scala2.12-java17-ubuntu/Dockerfile @@ -0,0 +1,79 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM eclipse-temurin:17-jre-jammy + +ARG spark_uid=185 + +RUN groupadd --system --gid=${spark_uid} spark && \ + useradd --system --uid=${spark_uid} --gid=spark spark + +RUN set -ex; \ + apt-get update; \ + apt-get install -y gnupg2 wget bash tini libc6 libpam-modules krb5-user libnss3 procps net-tools gosu libnss-wrapper; \ + mkdir -p /opt/spark; \ + mkdir /opt/spark/python; \ + mkdir -p /opt/spark/examples; \ + mkdir -p /opt/spark/work-dir; \ + chmod g+w /opt/spark/work-dir; \ + touch /opt/spark/RELEASE; \ + chown -R spark:spark /opt/spark; \ + echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \ + rm -rf /var/lib/apt/lists/* + +# Install Apache Spark +# https://downloads.apache.org/spark/KEYS +ENV SPARK_TGZ_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz \ + SPARK_TGZ_ASC_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz.asc \ + GPG_KEY=FC3AE3A7EAA1BAC98770840E7E1ABCC53AAA2216 + +RUN set -ex; \ + export SPARK_TMP="$(mktemp -d)"; \ + cd $SPARK_TMP; \ + wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \ + wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \ + gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \ + gpg --batch --verify spark.tgz.asc spark.tgz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" spark.tgz.asc; \ + \ + tar -xf spark.tgz --strip-components=1; \ + chown -R spark:spark .; \ + mv jars /opt/spark/; \ + mv bin /opt/spark/; \ + mv sbin /opt/spark/; \ + mv kubernetes/dockerfiles/spark/decom.sh /opt/; \ + mv examples /opt/spark/; \ + mv kubernetes/tests /opt/spark/; \ + mv data /opt/spark/; \ + mv python/pyspark /opt/spark/python/pyspark/; \ + mv python/lib /opt/spark/python/lib/; \ + mv R /opt/spark/; \ + chmod a+x /opt/decom.sh; \ + cd ..; \ + rm -rf "$SPARK_TMP"; + +COPY entrypoint.sh /opt/ + +ENV SPARK_HOME /opt/spark + +WORKDIR /opt/spark/work-dir + +USER spark + +ENTRYPOINT [ "/opt/entrypoint.sh" ] diff --git a/3.5.0/scala2.12-java17-ubuntu/entrypoint.sh b/3.5.0/scala2.12-java17-ubuntu/entrypoint.sh new file mode 100755 index 0000000..c576d8f --- /dev/null +++ b/3.5.0/scala2.12-java17-ubuntu/entrypoint.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Prevent any errors from being silently ignored +set -eo pipefail + +attempt_setup_fake_passwd_entry() { + # Check whether there is a passwd entry for the container UID + local myuid; myuid="$(id -u)" + # If there is no passwd entry for the container UID, attempt to fake one + # You can also refer to the https://github.com/docker-library/official-images/pull/13089#issuecomment-1534706523 + # It's to resolve OpenShift random UID case. + # See also: https://github.com/docker-library/postgres/pull/448 + if ! getent passwd "$myuid" &> /dev/null; then + local wrapper + for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do + if [ -s "$wrapper" ]; then + NSS_WRAPPER_PASSWD="$(mktemp)" + NSS_WRAPPER_GROUP="$(mktemp)" + export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + local mygid; mygid="$(id -g)" + printf 'spark:x:%s:%s:${SPARK_USER_NAME:-anonymous uid}:%s:/bin/false\n' "$myuid" "$mygid" "$SPARK_HOME" > "$NSS_WRAPPER_PASSWD" + printf 'spark:x:%s:\n' "$mygid" > "$NSS_WRAPPER_GROUP" + break + fi + done + fi +} + +if [ -z "$JAVA_HOME" ]; then + JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}') +fi + +SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*" +for v in "${!SPARK_JAVA_OPT_@}"; do + SPARK_EXECUTOR_JAVA_OPTS+=( "${!v}" ) +done + +if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then + SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH" +fi + +if ! [ -z "${PYSPARK_PYTHON+x}" ]; then + export PYSPARK_PYTHON +fi +if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then + export PYSPARK_DRIVER_PYTHON +fi + +# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor. +# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s. +if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then + export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)" +fi + +if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then + SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH"; +fi + +if ! [ -z "${SPARK_CONF_DIR+x}" ]; then + SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH"; +elif ! [ -z "${SPARK_HOME+x}" ]; then + SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH"; +fi + +# SPARK-43540: add current working directory into executor classpath +SPARK_CLASSPATH="$SPARK_CLASSPATH:$PWD" + +# Switch to spark if no USER specified (root by default) otherwise use USER directly +switch_spark_if_root() { + if [ $(id -u) -eq 0 ]; then + echo gosu spark + fi +} + +case "$1" in + driver) + shift 1 + CMD=( + "$SPARK_HOME/bin/spark-submit" + --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS" + --conf "spark.executorEnv.SPARK_DRIVER_POD_IP=$SPARK_DRIVER_BIND_ADDRESS" + --deploy-mode client + "$@" + ) + attempt_setup_fake_passwd_entry + # Execute the container CMD under tini for better hygiene + exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}" + ;; + executor) + shift 1 + CMD=( + ${JAVA_HOME}/bin/java + "${SPARK_EXECUTOR_JAVA_OPTS[@]}" + -Xms"$SPARK_EXECUTOR_MEMORY" + -Xmx"$SPARK_EXECUTOR_MEMORY" + -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH" + org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend + --driver-url "$SPARK_DRIVER_URL" + --executor-id "$SPARK_EXECUTOR_ID" + --cores "$SPARK_EXECUTOR_CORES" + --app-id "$SPARK_APPLICATION_ID" + --hostname "$SPARK_EXECUTOR_POD_IP" + --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID" + --podName "$SPARK_EXECUTOR_POD_NAME" + ) + attempt_setup_fake_passwd_entry + # Execute the container CMD under tini for better hygiene + exec $(switch_spark_if_root) /usr/bin/tini -s -- "${CMD[@]}" + ;; + + *) + # Non-spark-on-k8s command provided, proceeding in pass-through mode... + exec "$@" + ;; +esac diff --git a/add-dockerfiles.sh b/add-dockerfiles.sh index d61601e..5161c0c 100755 --- a/add-dockerfiles.sh +++ b/add-dockerfiles.sh @@ -26,7 +26,7 @@ # - Add 3.3.1 dockerfiles: # $ ./add-dockerfiles.sh 3.3.1 -VERSION=${1:-"3.3.0"} +VERSION=${1:-"3.5.0"} TAGS=" scala2.12-java11-python3-r-ubuntu @@ -35,6 +35,16 @@ scala2.12-java11-r-ubuntu scala2.12-java11-ubuntu " +# java17 images were added in 3.5.0. We need to skip java17 for 3.3.x and 3.4.x +if ! echo $VERSION | grep -Eq "^3.3|^3.4"; then + TAGS+=" + scala2.12-java17-python3-r-ubuntu + scala2.12-java17-python3-ubuntu + scala2.12-java17-r-ubuntu + scala2.12-java17-ubuntu + " +fi + for TAG in $TAGS; do OPTS="" if echo $TAG | grep -q "python"; then @@ -44,16 +54,23 @@ for TAG in $TAGS; do if echo $TAG | grep -q "r-"; then OPTS+=" --sparkr" fi - + + if echo $TAG | grep -q "java17"; then + OPTS+=" --java-version 17 --image eclipse-temurin:17-jre-jammy" + elif echo $TAG | grep -q "java11"; then + OPTS+=" --java-version 11 --image eclipse-temurin:11-jre-focal" + fi + OPTS+=" --spark-version $VERSION" mkdir -p $VERSION/$TAG - if [ "$TAG" == "scala2.12-java11-ubuntu" ]; then + if [ "$TAG" == "scala2.12-java11-ubuntu" ] || [ "$TAG" == "scala2.12-java17-ubuntu" ]; then python3 tools/template.py $OPTS > $VERSION/$TAG/Dockerfile python3 tools/template.py $OPTS -f entrypoint.sh.template > $VERSION/$TAG/entrypoint.sh chmod a+x $VERSION/$TAG/entrypoint.sh else python3 tools/template.py $OPTS -f r-python.template > $VERSION/$TAG/Dockerfile fi + done diff --git a/tools/ci_runner_cleaner/free_disk_space.sh b/tools/ci_runner_cleaner/free_disk_space.sh new file mode 100644 index 0000000..40a755c --- /dev/null +++ b/tools/ci_runner_cleaner/free_disk_space.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +echo "==================================" +echo "Free up disk space on CI system" +echo "==================================" + +echo "Listing 100 largest packages" +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 +df -h + +echo "Removing large packages" +sudo rm -rf /usr/share/dotnet/ +sudo rm -rf /usr/share/php/ +sudo rm -rf /usr/local/graalvm/ +sudo rm -rf /usr/local/.ghcup/ +sudo rm -rf /usr/local/share/powershell +sudo rm -rf /usr/local/share/chromium +sudo rm -rf /usr/local/lib/android +sudo rm -rf /usr/local/lib/node_modules + +sudo rm -rf /opt/az +sudo rm -rf /opt/hostedtoolcache/CodeQL +sudo rm -rf /opt/hostedtoolcache/go +sudo rm -rf /opt/hostedtoolcache/node + +sudo apt-get remove --purge -y '^aspnet.*' +sudo apt-get remove --purge -y '^dotnet-.*' +sudo apt-get remove --purge -y '^llvm-.*' +sudo apt-get remove --purge -y 'php.*' +sudo apt-get remove --purge -y '^mongodb-.*' +sudo apt-get remove --purge -y snapd google-chrome-stable microsoft-edge-stable firefox +sudo apt-get remove --purge -y azure-cli google-cloud-sdk mono-devel powershell libgl1-mesa-dri +sudo apt-get autoremove --purge -y +sudo apt-get clean + +df -h \ No newline at end of file diff --git a/tools/ci_runner_cleaner/free_disk_space_container.sh b/tools/ci_runner_cleaner/free_disk_space_container.sh new file mode 100644 index 0000000..3c6142d --- /dev/null +++ b/tools/ci_runner_cleaner/free_disk_space_container.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +echo "==================================" +echo "Free up disk space on CI system" +echo "==================================" + +echo "Listing 100 largest packages" +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 +df -h + +echo "Removing large packages" +rm -rf /__t/CodeQL +rm -rf /__t/go +rm -rf /__t/node + +df -h \ No newline at end of file diff --git a/tools/template.py b/tools/template.py index 7bd56d2..14903df 100755 --- a/tools/template.py +++ b/tools/template.py @@ -59,7 +59,7 @@ def parse_opts(): parser.add_argument( "-j", "--java-version", - help="The Spark version of Dockerfile.", + help="Java version of Dockerfile.", default="11", ) diff --git a/versions.json b/versions.json index 80d35b9..b6b32df 100644 --- a/versions.json +++ b/versions.json @@ -1,5 +1,34 @@ { "versions": [ + { + "path": "3.5.0/scala2.12-java17-python3-ubuntu", + "tags": [ + "3.5.0-scala2.12-java17-python3-ubuntu", + "3.5.0-java17-python3", + "3.5.0-java17", + "python3-java17" + ] + }, + { + "path": "3.5.0/scala2.12-java17-r-ubuntu", + "tags": [ + "3.5.0-scala2.12-java17-r-ubuntu", + "3.5.0-java17-r" + ] + }, + { + "path": "3.5.0/scala2.12-java17-ubuntu", + "tags": [ + "3.5.0-scala2.12-java17-ubuntu", + "3.5.0-java17-scala" + ] + }, + { + "path": "3.5.0/scala2.12-java17-python3-r-ubuntu", + "tags": [ + "3.5.0-scala2.12-java17-python3-r-ubuntu" + ] + }, { "path": "3.5.0/scala2.12-java11-python3-ubuntu", "tags": [ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org