KYLIN-1119 fix find_hive_dependency.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b62e73b7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b62e73b7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b62e73b7 Branch: refs/heads/2.x-staging Commit: b62e73b76a41dfa56f5f7fb8603dadc221827f75 Parents: 8970f22 Author: honma <ho...@ebay.com> Authored: Tue Nov 3 15:37:02 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Tue Nov 3 15:37:02 2015 +0800 ---------------------------------------------------------------------- build/bin/find-hive-dependency.sh | 17 ++++++++++++++--- .../kylin/engine/mr/common/AbstractHadoopJob.java | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b62e73b7/build/bin/find-hive-dependency.sh ---------------------------------------------------------------------- diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh index 98d0860..4982c93 100644 --- a/build/bin/find-hive-dependency.sh +++ b/build/bin/find-hive-dependency.sh @@ -19,10 +19,19 @@ do hive_conf_path=$data fi done -hive_home=`echo $hive_exec_path | awk -F '/lib/' '{print $1}'`/ -hcatalog=`find $hive_home -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'` -hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'` +# in some versions of hdp hcatalog is not in hive's classpath, find it separately +if [ -z "$HCAT_HOME" ] +then + echo "HCAT_HOME not found, try to find hcatalog path from hdp home" + hdp_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'` + hcatalog_home=${hdp_home}/hive-hcatalog +else + echo "HCAT_HOME is set to: $HCAT_HOME, use it to find hcatalog path:" + hcatalog_home=${HCAT_HOME} +fi + +hcatalog=`find ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'` if [ -z "$hcatalog" ] then @@ -30,6 +39,8 @@ then exit 1 fi + +hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'` hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog} echo "hive dependency: $hive_dependency" export hive_dependency http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b62e73b7/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index 1455664..ffa7c13 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -158,7 +158,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { String kylinHiveDependency = System.getProperty("kylin.hive.dependency"); String kylinHBaseDependency = System.getProperty("kylin.hbase.dependency"); - logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " and kylin.hive.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH); + logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " and kylin.hbase.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH); Configuration jobConf = job.getConfiguration(); String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);