This is an automated email from the ASF dual-hosted git repository.
angerszhuuuu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 734f14991 [CELEBORN-360] Add celeborn ratis shell command line (#1294)
734f14991 is described below
commit 734f14991a51ca3987b64154329d79d6e3ed79d6
Author: Angerszhuuuu <[email protected]>
AuthorDate: Thu Mar 2 16:30:45 2023 +0800
[CELEBORN-360] Add celeborn ratis shell command line (#1294)
* [CELEBORN-360] Add celeborn ratis shell command line
---
bin/celeborn-class | 14 +-----
bin/celeborn-ratis | 98 ++++++++++++++++++++++++++++++++++++
conf/ratis-log4j.properties.template | 31 ++++++++++++
sbin/celeborn-config.sh | 30 +++++++++++
sbin/celeborn-daemon.sh | 17 -------
5 files changed, 160 insertions(+), 30 deletions(-)
diff --git a/bin/celeborn-class b/bin/celeborn-class
index 9e6a5ba46..f98d196d5 100755
--- a/bin/celeborn-class
+++ b/bin/celeborn-class
@@ -36,18 +36,6 @@ if [ -z "$CELEBORN_ENV_LOADED" ]; then
fi
fi
-# Find the java binary
-if [ -n "${JAVA_HOME}" ]; then
- RUNNER="${JAVA_HOME}/bin/java"
-else
- if [ "$(command -v java)" ]; then
- RUNNER="java"
- else
- echo "JAVA_HOME is not set" >&2
- exit 1
- fi
-fi
-
# Find CELEBORN jars.
for i in "$@"
@@ -87,7 +75,7 @@ fi
# Turn off posix mode since it does not allow process substitution
set +o posix
CMD=()
-CMD+=("$RUNNER")
+CMD+=("$JAVA")
CMD=(${CMD[@]} "$CELEBORN_JAVA_OPTS")
CMD+=("-cp")
CMD+=("$CELEBORN_CLASSPATH")
diff --git a/bin/celeborn-ratis b/bin/celeborn-ratis
new file mode 100755
index 000000000..269e76da5
--- /dev/null
+++ b/bin/celeborn-ratis
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function printUsage {
+ echo "Usage: celeborn-ratis COMMAND [GENERIC_COMMAND_OPTIONS] [COMMAND_ARGS]"
+ echo
+ echo "COMMAND is one of:"
+ echo -e " sh \t Command line tool for Celeborn ratis"
+ echo
+ echo "GENERIC_COMMAND_OPTIONS supports:"
+ echo -e " -D<property=value>\t Use a value for a given ratis-shell property"
+ echo
+ echo "Commands print help when invoked without parameters."
+}
+
+function runJavaClass {
+ CLASS_ARGS=()
+ for arg in "$@"; do
+ case "${arg}" in
+ -D* | -X* | -agentlib* | -javaagent*)
+ CELEBORN_RATIS_SHELL_JAVA_OPTS+=" ${arg}" ;;
+ *)
+ CLASS_ARGS+=("${arg}")
+ esac
+ done
+ "${JAVA}" -cp "${CLASSPATH}" ${CELEBORN_RATIS_SHELL_JAVA_OPTS} "${CLASS}"
${PARAMETER} "${CLASS_ARGS[@]}"
+}
+
+function main {
+
+ if [[ $# == 0 ]]; then
+ printUsage
+ exit 1
+ fi
+
+ COMMAND=$1
+ shift
+
+ local CELEBORN_RATIS_SHELL_CLASSPATH
+
+ # load master jars since all ratis related in master
+ while read -d '' -r jarfile ; do
+ if [[ "$CELEBORN_RATIS_SHELL_CLASSPATH" == "" ]]; then
+ CELEBORN_RATIS_SHELL_CLASSPATH="$jarfile";
+ else
+
CELEBORN_RATIS_SHELL_CLASSPATH="$CELEBORN_RATIS_SHELL_CLASSPATH":"$jarfile"
+ fi
+ done < <(find "$CELEBORN_HOME/master-jars" ! -type d -name '*.jar' -print0 |
sort -z)
+
+
CELEBORN_RATIS_SHELL_CLIENT_CLASSPATH="${CELEBORN_CONF_DIR}/:${CELEBORN_RATIS_SHELL_CLASSPATH}"
+
+ if [ ! -f "${CELEBORN_CONF_DIR}/ratis-log4j.properties" ]; then
+ echo "${CELEBORN_CONF_DIR}/ratis-log4j.properties not exists!" >&2
+ fi
+
+ CELEBORN_RATIS_SHELL_JAVA_OPTS+=" -Dratis.shell.logs.dir=${CELEBORN_LOG_DIR}"
+ CELEBORN_RATIS_SHELL_JAVA_OPTS+="
-Dlog4j.configuration=file:${CELEBORN_CONF_DIR}/ratis-log4j.properties"
+ CELEBORN_RATIS_SHELL_JAVA_OPTS+="
-Dorg.apache.jasper.compiler.disablejsr199=true"
+ CELEBORN_RATIS_SHELL_JAVA_OPTS+=" -Djava.net.preferIPv4Stack=true"
+ CELEBORN_RATIS_SHELL_JAVA_OPTS+="
-Dorg.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads=false"
+
+ PARAMETER=""
+
+ case ${COMMAND} in
+ "sh")
+ CLASS="org.apache.ratis.shell.cli.sh.RatisShell"
+ CLASSPATH=${CELEBORN_RATIS_SHELL_CLIENT_CLASSPATH}
+ runJavaClass "$@"
+ ;;
+ *)
+ echo "Unsupported command ${COMMAND}" >&2
+ printUsage
+ exit 1
+ ;;
+ esac
+}
+
+# Load Celeborn related env
+if [ -z "${CELEBORN_HOME}" ]; then
+ export CELEBORN_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+. "${CELEBORN_HOME}/sbin/celeborn-config.sh"
+
+main "$@"
diff --git a/conf/ratis-log4j.properties.template
b/conf/ratis-log4j.properties.template
new file mode 100644
index 000000000..f75a6b1c4
--- /dev/null
+++ b/conf/ratis-log4j.properties.template
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# These properties may be overridden by system properties. log4j gives system
properties
+# a higher precedence than locally defined variables.
+log4j.rootLogger=INFO, SHELL_LOGGER
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2}
(%F:%M) - %m%n
+
+# SHELL Logger
+log4j.appender.SHELL_LOGGER=org.apache.log4j.RollingFileAppender
+log4j.appender.SHELL_LOGGER.File=${ratis.shell.logs.dir}/celeborn-ratis-shell.log
+log4j.appender.SHELL_LOGGER.MaxFileSize=100MB
+log4j.appender.SHELL_LOGGER.MaxBackupIndex=10
+log4j.appender.SHELL_LOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.SHELL_LOGGER.layout.ConversionPattern=%d{ISO8601} [%t] %-5p
%c{2} (%F:%M) - %m%n
diff --git a/sbin/celeborn-config.sh b/sbin/celeborn-config.sh
index a9ecd4f16..6382fb166 100755
--- a/sbin/celeborn-config.sh
+++ b/sbin/celeborn-config.sh
@@ -39,3 +39,33 @@ if [ -z "$CELEBORN_ENV_LOADED" ]; then
set +a
fi
fi
+
+# Find the java binary
+if [ -n "${JAVA_HOME}" ]; then
+ JAVA="${JAVA_HOME}/bin/java"
+else
+ if [ "$(command -v java)" ]; then
+ JAVA="java"
+ else
+ echo "JAVA_HOME is not set" >&2
+ exit 1
+ fi
+fi
+
+# Get log directory
+if [ "$CELEBORN_LOG_DIR" = "" ]; then
+ export CELEBORN_LOG_DIR="${CELEBORN_HOME}/logs"
+fi
+mkdir -p "$CELEBORN_LOG_DIR"
+touch "$CELEBORN_LOG_DIR"/.celeborn_test > /dev/null 2>&1
+TEST_LOG_DIR=$?
+if [ "${TEST_LOG_DIR}" = "0" ]; then
+ rm -f "$CELEBORN_LOG_DIR"/.celeborn_test
+else
+ chown "$CELEBORN_IDENT_STRING" "$CELEBORN_LOG_DIR"
+fi
+
+if [ "$CELEBORN_PID_DIR" = "" ]; then
+ CELEBORN_PID_DIR="${CELEBORN_HOME}/pids"
+fi
+
diff --git a/sbin/celeborn-daemon.sh b/sbin/celeborn-daemon.sh
index ce53cdcde..536de592c 100755
--- a/sbin/celeborn-daemon.sh
+++ b/sbin/celeborn-daemon.sh
@@ -90,23 +90,6 @@ if [ "$CELEBORN_IDENT_STRING" = "" ]; then
export CELEBORN_IDENT_STRING="$USER"
fi
-# get log directory
-if [ "$CELEBORN_LOG_DIR" = "" ]; then
- export CELEBORN_LOG_DIR="${CELEBORN_HOME}/logs"
-fi
-mkdir -p "$CELEBORN_LOG_DIR"
-touch "$CELEBORN_LOG_DIR"/.celeborn_test > /dev/null 2>&1
-TEST_LOG_DIR=$?
-if [ "${TEST_LOG_DIR}" = "0" ]; then
- rm -f "$CELEBORN_LOG_DIR"/.celeborn_test
-else
- chown "$CELEBORN_IDENT_STRING" "$CELEBORN_LOG_DIR"
-fi
-
-if [ "$CELEBORN_PID_DIR" = "" ]; then
- CELEBORN_PID_DIR="${CELEBORN_HOME}/pids"
-fi
-
# some variables
log="$CELEBORN_LOG_DIR/celeborn-$CELEBORN_IDENT_STRING-$command-$instance-$HOSTNAME.out"
pid="$CELEBORN_PID_DIR/celeborn-$CELEBORN_IDENT_STRING-$command-$instance.pid"
