This is an automated email from the ASF dual-hosted git repository. nic pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push: new a471808 KYLIN-3698 some enhancement for check-env.sh a471808 is described below commit a471808626ee9349173255d329673c509b5f4037 Author: yaqian.zhang <598593...@qq.com> AuthorDate: Wed May 20 18:37:46 2020 +0800 KYLIN-3698 some enhancement for check-env.sh --- build/bin/check-env.sh | 38 ++++++++++++++++------------------- build/bin/find-flink-dependency.sh | 3 ++- build/bin/find-kafka-dependency.sh | 5 ++++- build/bin/find-spark-dependency.sh | 41 ++++++++++++++++++++++++++++---------- build/bin/kylin.sh | 6 ------ 5 files changed, 54 insertions(+), 39 deletions(-) diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh index a6250ed..e3b61c7 100755 --- a/build/bin/check-env.sh +++ b/build/bin/check-env.sh @@ -83,35 +83,31 @@ else echo "...................................................[`setColor 32 PASS`]" fi -source ${dir}/find-spark-dependency.sh -if [[ $? == 0 ]]; then + +bash ${dir}/find-spark-dependency.sh +rtn=$? +if [[ $rtn == 0 ]]; then echo "...................................................[`setColor 32 PASS`]" +elif [[ $rtn == 3 ]]; then + echo "...................................................[`setColor 33 SKIP`]" fi -source ${dir}/find-flink-dependency.sh -if [[ $? == 0 ]]; then +bash ${dir}/find-flink-dependency.sh +rtn=$? +if [[ $rtn == 0 ]]; then echo "...................................................[`setColor 32 PASS`]" +elif [[ $rtn == 3 ]]; then + echo "...................................................[`setColor 33 SKIP`]" fi -SPARK_EVENTLOG_DIR=`bash $KYLIN_HOME/bin/get-properties.sh kylin.engine.spark-conf.spark.eventLog.dir` -if [ -n "$SPARK_EVENTLOG_DIR" ] -then - hadoop ${hadoop_conf_param} fs -mkdir -p $SPARK_EVENTLOG_DIR - if [ $? != 0 ] - then - quit "Failed to create $SPARK_EVENTLOG_DIR. Please make sure the user has right to access $SPARK_EVENTLOG_DIR" - fi +bash ${dir}/find-kafka-dependency.sh +rtn=$? +if [[ $rtn == 0 ]]; then + echo "...................................................[`setColor 32 PASS`]" +elif [[ $rtn == 3 ]]; then + echo "...................................................[`setColor 33 SKIP`]" fi -SPARK_HISTORYLOG_DIR=`bash $KYLIN_HOME/bin/get-properties.sh kylin.engine.spark-conf.spark.history.fs.logDirectory` -if [ -n "$SPARK_HISTORYLOG_DIR" ] -then - hadoop ${hadoop_conf_param} fs -mkdir -p $SPARK_HISTORYLOG_DIR - if [ $? != 0 ] - then - quit "Failed to create $SPARK_HISTORYLOG_DIR. Please make sure the user has right to access $SPARK_HISTORYLOG_DIR" - fi -fi ${KYLIN_HOME}/bin/check-port-availability.sh || exit 1; diff --git a/build/bin/find-flink-dependency.sh b/build/bin/find-flink-dependency.sh index 4485730..10f3643 100755 --- a/build/bin/find-flink-dependency.sh +++ b/build/bin/find-flink-dependency.sh @@ -37,8 +37,9 @@ fi if [ ! -d "$flink_home/lib" ] then - echo "Optional dependency flink not found, if you need this; set FLINK_HOME, or run bin/download-flink.sh" + echo `setColor 33 "Optional dependency flink not found, if you need this; set FLINK_HOME, or run bin/download-flink.sh"` echo "echo 'skip flink_dependency'" > ${dir}/cached-flink-dependency.sh + exit 3 else flink_dependency=`find -L $flink_home/lib -name '*.jar' ! -name '*shaded-hadoop*' ! -name 'kafka*' ! -name '*log4j*' ! -name '*slf4j*' ! -name '*calcite*' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'` if [ -z "$flink_dependency" ] diff --git a/build/bin/find-kafka-dependency.sh b/build/bin/find-kafka-dependency.sh index b12e6a4..4782680 100755 --- a/build/bin/find-kafka-dependency.sh +++ b/build/bin/find-kafka-dependency.sh @@ -19,13 +19,16 @@ source ${KYLIN_HOME:-"$(cd -P -- "$(dirname -- "$0")" && pwd -P)/../"}/bin/header.sh +echo Retrieving Kafka dependency... + kafka_home= echo Retrieving kafka dependency... if [ -z "$KAFKA_HOME" ] then - verbose "Couldn't find kafka home. If you want to enable streaming processing, Please set KAFKA_HOME to the path which contains kafka dependencies." + echo "Couldn't find kafka home. If you want to enable streaming processing, Please set KAFKA_HOME to the path which contains kafka dependencies." + exit 3 else verbose "KAFKA_HOME is set to: $KAFKA_HOME, use it to locate kafka dependencies." kafka_home=$KAFKA_HOME diff --git a/build/bin/find-spark-dependency.sh b/build/bin/find-spark-dependency.sh index 368cc0d..2321cea 100755 --- a/build/bin/find-spark-dependency.sh +++ b/build/bin/find-spark-dependency.sh @@ -35,18 +35,39 @@ then spark_home=$KYLIN_HOME/spark fi -if [ ! -d "$spark_home/jars" ] - then - quit "spark not found, set SPARK_HOME, or run bin/download-spark.sh" +SPARK_EVENTLOG_DIR=`bash $KYLIN_HOME/bin/get-properties.sh kylin.engine.spark-conf.spark.eventLog.dir` +if [ -n "$SPARK_EVENTLOG_DIR" ] +then + hadoop ${hadoop_conf_param} fs -mkdir -p $SPARK_EVENTLOG_DIR + if [ $? != 0 ] + then + quit "Failed to create $SPARK_EVENTLOG_DIR. Please make sure the user has right to access $SPARK_EVENTLOG_DIR" + fi fi -spark_dependency=`find -L $spark_home/jars -name '*.jar' ! -name '*slf4j*' ! -name '*calcite*' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'` -if [ -z "$spark_dependency" ] +SPARK_HISTORYLOG_DIR=`bash $KYLIN_HOME/bin/get-properties.sh kylin.engine.spark-conf.spark.history.fs.logDirectory` +if [ -n "$SPARK_HISTORYLOG_DIR" ] then - quit "spark jars not found" -else - verbose "spark dependency: $spark_dependency" - export spark_dependency + hadoop ${hadoop_conf_param} fs -mkdir -p $SPARK_HISTORYLOG_DIR + if [ $? != 0 ] + then + quit "Failed to create $SPARK_HISTORYLOG_DIR. Please make sure the user has right to access $SPARK_HISTORYLOG_DIR" + fi fi -echo "export spark_dependency=$spark_dependency" > ${dir}/cached-spark-dependency.sh +if [ ! -d "$spark_home/jars" ] + then + echo `setColor 33 "Optional dependency spark not found, if you need this; set SPARK_HOME, or run bin/download-spark.sh"` + echo "echo 'skip spark_dependency'" > ${dir}/cached-spark-dependency.sh + exit 3 + else + spark_dependency=`find -L $spark_home/jars -name '*.jar' ! -name '*slf4j*' ! -name '*calcite*' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'` + if [ -z "$spark_dependency" ] + then + quit "spark jars not found" + else + verbose "spark dependency: $spark_dependency" + export spark_dependency + fi + echo "export spark_dependency=$spark_dependency" > ${dir}/cached-spark-dependency.sh +fi diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh index d56976b..59d6de7 100755 --- a/build/bin/kylin.sh +++ b/build/bin/kylin.sh @@ -40,16 +40,10 @@ function retrieveDependency() { source ${dir}/cached-hive-dependency.sh source ${dir}/cached-hbase-dependency.sh source ${dir}/cached-hadoop-conf-dir.sh - source ${dir}/cached-kafka-dependency.sh - source ${dir}/cached-spark-dependency.sh - source ${dir}/cached-flink-dependency.sh else source ${dir}/find-hive-dependency.sh source ${dir}/find-hbase-dependency.sh source ${dir}/find-hadoop-conf-dir.sh - source ${dir}/find-kafka-dependency.sh - source ${dir}/find-spark-dependency.sh - source ${dir}/find-flink-dependency.sh fi #retrive $KYLIN_EXTRA_START_OPTS