This is an automated email from the ASF dual-hosted git repository.
xianjin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new e20fb622 [#600] chore(operator): change JDK base from openjdk to
eclipse-temurin (#617)
e20fb622 is described below
commit e20fb6225b30fbd171222216ec8163777d0f8317
Author: advancedxy <[email protected]>
AuthorDate: Tue Feb 21 10:33:27 2023 +0800
[#600] chore(operator): change JDK base from openjdk to eclipse-temurin
(#617)
### What changes were proposed in this pull request?
1. replace openjdk with eclipse-temurin
2. split rss-server images into two parts: base and upper
3. fix lsof location in start.sh
4. enable rss-server image build on ci
### Why are the changes needed?
Fixes #600
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Manually verified
---
.github/workflows/build.yml | 4 +--
.github/workflows/changes.yml | 6 ++++
build_distribution.sh | 2 +-
deploy/kubernetes/docker/Dockerfile | 6 ++--
.../kubernetes/docker/base/centos/Dockerfile | 29 ++++-----------
.../kubernetes/docker/base/debian/Dockerfile | 29 ++++-----------
deploy/kubernetes/docker/build.sh | 41 +++++++++++++++++-----
deploy/kubernetes/docker/start.sh | 4 +--
deploy/kubernetes/pom.xml | 13 +++++++
pom.xml | 1 +
10 files changed, 71 insertions(+), 64 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 41c00c93..6d12d3b6 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -82,6 +82,6 @@ jobs:
if: needs.changes.outputs.kubernetes == 'true' || github.event_name ==
'push'
uses: ./.github/workflows/single.yml
with:
- maven-args: package -Pkubernetes -DskipUTs -DskipITs
+ maven-args: package -Pkubernetes -DskipUTs -DskipITs
-DskipBuildImage=${{ needs.changes.outputs.rss_server_docker == 'false' }}
cache-key: package
- go-version: '1.17'
+ go-version: '1.17'
\ No newline at end of file
diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml
index 318e0194..9dd6a205 100644
--- a/.github/workflows/changes.yml
+++ b/.github/workflows/changes.yml
@@ -23,6 +23,9 @@ on:
kubernetes:
description: "whether current workflow touches deploy/kubernetes dir
or not"
value: ${{ jobs.changes.outputs.kubernetes }}
+ rss_server_docker:
+ description: "whether current workflow touches
deploy/kubernetes/docker dir or not"
+ value: ${{ jobs.changes.outputs.rss_server_docker }}
jobs:
changes:
@@ -37,5 +40,8 @@ jobs:
filters: |
kubernetes:
- 'deploy/kubernetes/**'
+ rss_server_docker:
+ - 'deploy/kubernetes/docker/**'
outputs:
kubernetes: ${{ steps.filter.outputs.kubernetes }}
+ rss_server_docker: ${{ steps.filter.outputs.rss_server_docker }}
diff --git a/build_distribution.sh b/build_distribution.sh
index 2561807c..7a853435 100755
--- a/build_distribution.sh
+++ b/build_distribution.sh
@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
diff --git a/deploy/kubernetes/docker/Dockerfile
b/deploy/kubernetes/docker/Dockerfile
index 8a3c591e..df40dc88 100644
--- a/deploy/kubernetes/docker/Dockerfile
+++ b/deploy/kubernetes/docker/Dockerfile
@@ -15,7 +15,8 @@
# limitations under the License.
#
-FROM openjdk:8-jdk-slim
+ARG BASE_IMAGE=uniffle-base:latest
+FROM ${BASE_IMAGE}
ARG HADOOP_VERSION
ARG RSS_VERSION
@@ -27,9 +28,6 @@ LABEL author=${AUTHOR}
LABEL git-branch=${GIT_BRANCH}
LABEL git-commit=${GIT_COMMIT}
-RUN apt-get update && apt-get install -y zlib1g zlib1g-dev lzop lsof netcat
dnsutils less procps iputils-ping \
- && apt-get clean && rm -rf /var/lib/apt/lists/*
-
RUN useradd -ms /bin/bash rssadmin
RUN mkdir -p /data/rssadmin/
RUN chown -R rssadmin:rssadmin /data
diff --git a/.github/workflows/changes.yml
b/deploy/kubernetes/docker/base/centos/Dockerfile
similarity index 57%
copy from .github/workflows/changes.yml
copy to deploy/kubernetes/docker/base/centos/Dockerfile
index 318e0194..5a62e64e 100644
--- a/.github/workflows/changes.yml
+++ b/deploy/kubernetes/docker/base/centos/Dockerfile
@@ -15,27 +15,10 @@
# limitations under the License.
#
-name: Change detection
+# this docker file serves the rss server base image for CentOS/RedHat
distributions
+ARG BASE_IMAGE=eclipse-temurin:8-jdk-centos7
+FROM ${BASE_IMAGE}
-on:
- workflow_call:
- outputs:
- kubernetes:
- description: "whether current workflow touches deploy/kubernetes dir
or not"
- value: ${{ jobs.changes.outputs.kubernetes }}
-
-jobs:
- changes:
- runs-on: ubuntu-20.04
- name: detect
- steps:
- - name: Checkout project
- uses: actions/checkout@v3
- - uses: dorny/paths-filter@v2
- id: filter
- with:
- filters: |
- kubernetes:
- - 'deploy/kubernetes/**'
- outputs:
- kubernetes: ${{ steps.filter.outputs.kubernetes }}
+RUN yum install lzo zlib zlib-devel lzop lsof netcat dnsutils less procps
iputils-ping -y && \
+ yum clean all && \
+ rm -rf /var/cache/yum
diff --git a/.github/workflows/changes.yml
b/deploy/kubernetes/docker/base/debian/Dockerfile
similarity index 57%
copy from .github/workflows/changes.yml
copy to deploy/kubernetes/docker/base/debian/Dockerfile
index 318e0194..6c639f91 100644
--- a/.github/workflows/changes.yml
+++ b/deploy/kubernetes/docker/base/debian/Dockerfile
@@ -15,27 +15,10 @@
# limitations under the License.
#
-name: Change detection
+# this docker file serves the rss server base image for Debian/Ubuntu releated
distributions
+ARG BASE_IMAGE=eclipse-temurin:8-jdk
+FROM ${BASE_IMAGE}
-on:
- workflow_call:
- outputs:
- kubernetes:
- description: "whether current workflow touches deploy/kubernetes dir
or not"
- value: ${{ jobs.changes.outputs.kubernetes }}
-
-jobs:
- changes:
- runs-on: ubuntu-20.04
- name: detect
- steps:
- - name: Checkout project
- uses: actions/checkout@v3
- - uses: dorny/paths-filter@v2
- id: filter
- with:
- filters: |
- kubernetes:
- - 'deploy/kubernetes/**'
- outputs:
- kubernetes: ${{ steps.filter.outputs.kubernetes }}
+RUN apt-get update && \
+ apt-get install -y zlib1g zlib1g-dev lzop lsof netcat dnsutils less procps
iputils-ping && \
+ apt-get clean && rm -rf /var/lib/apt/lists/*
diff --git a/deploy/kubernetes/docker/build.sh
b/deploy/kubernetes/docker/build.sh
old mode 100644
new mode 100755
index a318b860..aca37dd5
--- a/deploy/kubernetes/docker/build.sh
+++ b/deploy/kubernetes/docker/build.sh
@@ -24,18 +24,20 @@ function exit_with_usage() {
echo "Usage:"
echo
"+------------------------------------------------------------------------------------------------------+"
echo "| ./build.sh [--hadoop-version <hadoop version>] [--registry <registry
url>] [--author <author name>] |"
- echo "| [--apache-mirror <apache mirror url>]
|"
+ echo "| [--base-os-distribution <os distribution>] [--base-image
<base image url>] |"
+ echo "| [--push-image <true|false>] [--apache-mirror <apache
mirror url>] |"
echo
"+------------------------------------------------------------------------------------------------------+"
exit 1
}
-UNKNOWN_REGISTRY="UNKNOWN_REGISTRY"
-
-REGISTRY=$UNKNOWN_REGISTRY
+REGISTRY="docker.io/library"
HADOOP_VERSION=2.8.5
AUTHOR=$(whoami)
# If you are based in China, you could pass --apache-mirror <a_mirror_url>
when building this.
APACHE_MIRROR="https://dlcdn.apache.org";
+OS_DISTRIBUTION=debian
+BASE_IMAGE=""
+PUSH_IMAGE="true"
while (( "$#" )); do
case $1 in
@@ -51,6 +53,18 @@ while (( "$#" )); do
AUTHOR="$2"
shift
;;
+ --base-os-distribution)
+ OS_DISTRIBUTION="$2"
+ shift
+ ;;
+ --base-image)
+ BASE_IMAGE="$2"
+ shift
+ ;;
+ --push-image)
+ PUSH_IMAGE="$2"
+ shift
+ ;;
--help)
exit_with_usage
;;
@@ -73,10 +87,16 @@ while (( "$#" )); do
shift
done
-if [ "$REGISTRY" == $UNKNOWN_REGISTRY ]; \
- then echo "please set registry url"; exit; \
+if [ -z "$BASE_IMAGE" ]; then
+ echo "start building base image: uniffle-base"
+ docker build -t "uniffle-base:latest" \
+ -f base/"${OS_DISTRIBUTION}"/Dockerfile .
+ BASE_IMAGE="uniffle-base:latest"
+else
+ echo "using base image(${BASE_IMAGE}) to build rss server"
fi
+
HADOOP_FILE=hadoop-${HADOOP_VERSION}.tar.gz
ARCHIVE_HADOOP_URL=https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
HADOOP_URL=${APACHE_MIRROR}/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
@@ -91,7 +111,7 @@ cd $RSS_DIR || exit
RSS_VERSION=$(mvn help:evaluate -Dexpression=project.version 2>/dev/null |
grep -v "INFO" | grep -v "WARNING" | tail -n 1)
RSS_FILE=rss-${RSS_VERSION}.tgz
if [ ! -e "$RSS_FILE" ]; \
- then sh ./build_distribution.sh; \
+ then bash ./build_distribution.sh; \
else echo "$RSS_FILE has been built"; \
fi
cd "$OLDPWD" || exit
@@ -108,7 +128,10 @@ docker build -t "$IMAGE" \
--build-arg AUTHOR="$AUTHOR" \
--build-arg GIT_COMMIT="$GIT_COMMIT" \
--build-arg GIT_BRANCH="$GIT_BRANCH" \
+ --build-arg BASE_IMAGE="$BASE_IMAGE" \
-f Dockerfile --no-cache .
-echo "pushing image: $IMAGE"
-docker push "$IMAGE"
+if [ x"${PUSH_IMAGE}" == x"true" ]; then
+ echo "pushing image: $IMAGE"
+ docker push "$IMAGE"
+fi
diff --git a/deploy/kubernetes/docker/start.sh
b/deploy/kubernetes/docker/start.sh
index d7ca5fb2..c2ae3858 100644
--- a/deploy/kubernetes/docker/start.sh
+++ b/deploy/kubernetes/docker/start.sh
@@ -31,7 +31,7 @@ if [ "$SERVICE_NAME" == "coordinator" ];then
bash ${basedir}/bin/start-coordinator.sh &
sleep 10
while : ; do
- pid=$(/usr/bin/lsof -i:"${COORDINATOR_RPC_PORT}" -sTCP:LISTEN)
+ pid=$(lsof -i:"${COORDINATOR_RPC_PORT}" -sTCP:LISTEN)
if [ "$pid" = "" ]; then
break
else
@@ -46,7 +46,7 @@ if [ "$SERVICE_NAME" == "server" ];then
bash ${basedir}/bin/start-shuffle-server.sh &
sleep 10
while : ; do
- pid=$(/usr/bin/lsof -i:"$SERVER_RPC_PORT" -sTCP:LISTEN)
+ pid=$(lsof -i:"$SERVER_RPC_PORT" -sTCP:LISTEN)
if [ "$pid" = "" ]; then
break
else
diff --git a/deploy/kubernetes/pom.xml b/deploy/kubernetes/pom.xml
index b2f745b7..cbc81345 100644
--- a/deploy/kubernetes/pom.xml
+++ b/deploy/kubernetes/pom.xml
@@ -45,6 +45,19 @@
<executable>${basedir}/build-operator.sh</executable>
</configuration>
</execution>
+ <execution>
+ <id>Build RSS server image</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <workingDirectory>${basedir}/docker</workingDirectory>
+ <executable>${basedir}/docker/build.sh</executable>
+ <commandlineArgs>--push-image false --hadoop-version
2.10.2</commandlineArgs>
+ <skip>${skipBuildImage}</skip>
+ </configuration>
+ </execution>
<execution>
<id>Test Kubernetes Operator</id>
<phase>test</phase>
diff --git a/pom.xml b/pom.xml
index 84597b6d..d6cd21ad 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,6 +81,7 @@
<trimStackTrace>false</trimStackTrace>
<skipUTs>${skipTests}</skipUTs>
<skipITs>${skipTests}</skipITs>
+ <skipBuildImage>true</skipBuildImage>
</properties>
<repositories>
