This is an automated email from the ASF dual-hosted git repository. zhouky pushed a commit to branch branch-0.2 in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
commit 30e9e9bf9eb3a4704e4bc522ed667129f0104638 Author: Cheng Pan <[email protected]> AuthorDate: Tue Dec 20 15:57:14 2022 +0800 [CELEBORN-151][K8S] Celeborn should run as celeborn instead of root in container (#895) --- docker/DEPLOY_ON_K8S.md | 59 +++++++++++++++------------ docker/Dockerfile | 21 +++++++++- docker/helm/templates/configmap.yaml | 6 +-- docker/helm/templates/master-statefulset.yaml | 2 + docker/helm/templates/worker-statefulset.yaml | 2 + docker/helm/values.yaml | 23 ++++++----- 6 files changed, 72 insertions(+), 41 deletions(-) diff --git a/docker/DEPLOY_ON_K8S.md b/docker/DEPLOY_ON_K8S.md index 1ad417b4..5e3c45b0 100644 --- a/docker/DEPLOY_ON_K8S.md +++ b/docker/DEPLOY_ON_K8S.md @@ -1,33 +1,41 @@ -# How to Deploy RSS on Kubernetes +# How to Deploy Celeborn on Kubernetes ## Prerequisite -Celeborn is recommended to be deployed on nodes with local disk. Before starting, please make sure local disks on nodes are mounted to specific path. +Celeborn is recommended to be deployed on nodes with local disk. Before starting, please make sure +local disks on nodes are mounted to specific path. ## [Optional] Build Celeborn docker image -We have provided a docker image for Celeborn in helm chart. If you want to build your own Celeborn image, run docker build with our Dockerfile. +We have provided a docker image for Celeborn in helm chart. If you want to build your own Celeborn +image, run docker build with our Dockerfile. -You should download or build your binary package first, then decompress it, cd decompress directory, use following command to build docker image. +You should download or build your binary package first, then decompress it, cd decompress directory, +use following command to build docker image. -` -docker build -f docker/Dockerfile -t ${your-repo}:${tag} . -` +``` +docker build . \ + --file docker/Dockerfile \ + --build-arg celeborn_uid=10006 \ + --build-arg celeborn_gid=10006 \ + --tag ${your-repo}:${tag} +``` ## Deploy Celeborn with helm #### Install kubectl and Helm -Please install and config kubectl and Helm first. See [Installing kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) and [Installing Helm](https://helm.sh/docs/intro/install/). +Please install and config kubectl and Helm first. See [Installing kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) +and [Installing Helm](https://helm.sh/docs/intro/install/). -#### Create namespace on kubernetes -` -kubectl create ns rss -` +#### Create namespace on Kubernetes +``` +kubectl create namespace celeborn +``` #### [Optional] Modify helm values file values.yaml -You can modify helm values file and set up customed deployment configuration. -` +You can modify helm values file and set up customized deployment configuration. +``` vim ${CELEBORN_HOME}/docker/helm/values.yaml -` +``` These values are suggested to be checked before deploy: - masterReplicas (number of Celeborn Master) - workerReplicas (number of Celeborn Worker) @@ -37,17 +45,18 @@ These values are suggested to be checked before deploy: For more information of Celeborn configurations, see [CONFIGURATIONS](../CONFIGURATION_GUIDE.md) #### Install Celeborn -` -helm install celeborn-helm ${CELEBORN_HOME}/docker/helm -n ${celeborn namespace} -` +``` +helm install celeborn ${CELEBORN_HOME}/docker/helm -n ${celeborn namespace} +``` #### Connect to Celeborn in K8s pod -After installation, you can connect to Celeborn master through headless service. For example, this is the spark configuration for 3-master RSS: -` -spark.celeborn.master.endpoints=shuffleservice-master-0.rss-master-svc.${celeborn namespace},shuffleservice-master-1.rss-master-svc.${celeborn namespace},shuffleservice-master-2.rss-master-svc.${celeborn namespace} -` +After installation, you can connect to Celeborn master through headless service. For example, +this is the spark configuration for 3-master Celeborn: +``` +spark.celeborn.master.endpoints=celeborn-master-0.celeborn-master-svc.${celeborn namespace},celeborn-master-1.celeborn-master-svc.${celeborn namespace},celeborn-master-2.celeborn-master-svc.${celeborn namespace} +``` #### Uninstall Celeborn -` -helm uninstall celeborn-helm -n ${celeborn namespace} -` \ No newline at end of file +``` +helm uninstall celeborn -n ${celeborn namespace} +``` \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index f799b8d4..3a81c2ee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,14 +18,22 @@ ARG java_image_tag=8-jdk-focal FROM eclipse-temurin:${java_image_tag} -USER root +ARG celeborn_uid=10006 +ARG celeborn_gid=10006 + +ENV CELEBORN_HOME=/opt/celeborn +ENV PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/busybox:${CELEBORN_HOME}/sbin:${CELEBORN_HOME}/bin RUN set -ex && \ apt-get update && \ apt-get install -y bash tini busybox bind9-utils telnet net-tools procps krb5-user dnsutils && \ ln -snf /bin/bash /bin/sh && \ rm -rf /var/cache/apt/* && \ - mkdir -p /opt/celeborn + mkdir /opt/busybox && \ + busybox --install /opt/busybox && \ + groupadd --gid=${celeborn_gid} celeborn && \ + useradd --uid=${celeborn_uid} --gid=${celeborn_gid} celeborn -d /home/celeborn -m && \ + mkdir -p ${CELEBORN_HOME} COPY bin /opt/celeborn/bin COPY sbin /opt/celeborn/sbin @@ -34,3 +42,12 @@ COPY jars /opt/celeborn/jars COPY master-jars /opt/celeborn/master-jars COPY worker-jars /opt/celeborn/worker-jars COPY RELEASE /opt/celeborn/RELEASE + +RUN chown -R celeborn:celeborn ${CELEBORN_HOME} && \ + chmod -R ug+rw ${CELEBORN_HOME} && \ + chmod a+x ${CELEBORN_HOME}/bin/* && \ + chmod a+x ${CELEBORN_HOME}/sbin/* + +USER celeborn + +ENTRYPOINT ["/usr/bin/tini", "--"] diff --git a/docker/helm/templates/configmap.yaml b/docker/helm/templates/configmap.yaml index d6e40c2f..7a6a5bac 100644 --- a/docker/helm/templates/configmap.yaml +++ b/docker/helm/templates/configmap.yaml @@ -31,9 +31,9 @@ data: {{- end }} {{- range $key, $val := .Values.celeborn }} {{ $key }}={{ $val }} - {{- end -}} - - rss-env.sh: | + {{- end }} + + celeborn-env.sh: | {{- range $key, $val := .Values.environments }} {{ $key }}="{{ $val }}" {{- end}} diff --git a/docker/helm/templates/master-statefulset.yaml b/docker/helm/templates/master-statefulset.yaml index 579f9a94..83de610a 100644 --- a/docker/helm/templates/master-statefulset.yaml +++ b/docker/helm/templates/master-statefulset.yaml @@ -58,6 +58,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + securityContext: + fsGroup: {{ .Values.gid }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" diff --git a/docker/helm/templates/worker-statefulset.yaml b/docker/helm/templates/worker-statefulset.yaml index 798fb308..4b407880 100644 --- a/docker/helm/templates/worker-statefulset.yaml +++ b/docker/helm/templates/worker-statefulset.yaml @@ -58,6 +58,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + securityContext: + fsGroup: {{ .Values.gid }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" diff --git a/docker/helm/values.yaml b/docker/helm/values.yaml index 63c4bbb0..33323512 100644 --- a/docker/helm/values.yaml +++ b/docker/helm/values.yaml @@ -36,27 +36,28 @@ workerReplicas: 5 # celeborn release version celebornVersion: 0.1.1 +gid: 10006 + # celeborn configurations celeborn: - # please update celeborn.worker.storage.dirs to disk mount path on k8s node + celeborn.ha.master.ratis.raft.server.storage.dir: /mnt/rss_ratis/ celeborn.worker.storage.dirs: /mnt/disk1,/mnt/disk2,/mnt/disk3,/mnt/disk4 celeborn.push.replicate.enabled: true + celeborn.metrics.enabled: true celeborn.master.metrics.prometheus.port: 9098 + celeborn.worker.metrics.prometheus.port: 9096 celeborn.worker.monitor.disk.enabled: false - rss.rpc.io.serverThreads: 64 - rss.worker.fetch.chunk.size: 8m - rss.rpc.io.numConnectionsPerPeer: 2 + celeborn.worker.fetch.chunk.size: 8m + celeborn.rpc.io.serverThreads: 64 + celeborn.rpc.io.numConnectionsPerPeer: 2 + celeborn.rpc.io.clientThreads: 64 + celeborn.rpc.dispatcher.numThreads: 4 celeborn.worker.flush.buffer.size: 256K - celeborn.metrics.enabled: true - rss.push.io.threads: 32 celeborn.worker.fetch.io.threads: 32 + celeborn.push.io.threads: 32 celeborn.push.stageEnd.timeout: 120s - celeborn.worker.metrics.prometheus.port: 9096 - rss.rpc.io.clientThreads: 64 celeborn.application.heartbeat.timeout: 120s - rss.rpc.dispatcher.numThreads: 4 celeborn.worker.heartbeat.timeout: 120s - celeborn.ha.master.ratis.raft.server.storage.dir: /mnt/rss_ratis/ environments: CELEBORN_MASTER_MEMORY: 2g @@ -64,7 +65,7 @@ environments: CELEBORN_WORKER_MEMORY: 2g CELEBORN_WORKER_OFFHEAP_MEMORY: 12g CELEBORN_WORKER_JAVA_OPTS: "-XX:-PrintGC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:gc-worker.out -Dio.netty.leakDetectionLevel=advanced" - CELEBORN_NO_DAEMONIZE: "yes" + CELEBORN_NO_DAEMONIZE: 1 TZ: "Asia/Shanghai" podMonitor:
