AMBARI-18383 : Oozie spark failed due to incomplete spark share-lib directory. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1e21842e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1e21842e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1e21842e Branch: refs/heads/branch-dev-patch-upgrade Commit: 1e21842e3ac9e2b6a4552cb0d58294a22d3c2e40 Parents: 0855174 Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Tue Sep 13 16:21:33 2016 -0700 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Tue Sep 13 16:21:33 2016 -0700 ---------------------------------------------------------------------- .../4.0.0.2.0/package/scripts/oozie_service.py | 11 ++++ .../main/resources/scripts/Ambaripreupload.py | 54 ++++++++++++++++++-- 2 files changed, 62 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1e21842e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py index 8d80638..4740131 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py @@ -122,6 +122,17 @@ def oozie_service(action = 'start', upgrade_type=None): if params.host_sys_prepped: print "Skipping creation of oozie sharelib as host is sys prepped" + # Copy current hive-site to hdfs:/user/oozie/share/lib/spark/ + params.HdfsResource(format("{hdfs_share_dir}/lib/spark/hive-site.xml"), + action="create_on_execute", + type = 'file', + mode=0444, + owner=params.oozie_user, + group=params.user_group, + source=format("{hive_conf_dir}/hive-site.xml"), + ) + params.HdfsResource(None, action="execute") + hdfs_share_dir_exists = True # skip time-expensive hadoop fs -ls check elif WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): # check with webhdfs is much faster than executing hadoop fs -ls. http://git-wip-us.apache.org/repos/asf/ambari/blob/1e21842e/ambari-server/src/main/resources/scripts/Ambaripreupload.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py index 7c8c601..9492e17 100644 --- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py +++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py @@ -37,7 +37,7 @@ from resource_management.core import shell from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import Execute, Directory from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format from resource_management.libraries.functions.oozie_prepare_war import prepare_war @@ -327,7 +327,55 @@ with Environment() as env: action="delete_on_execute", type = 'directory' ) - + + spark_client_dir = format("/usr/hdp/{stack_version}/spark") + + if os.path.exists(spark_client_dir): + # Rename /usr/hdp/{stack_version}/oozie/share/lib/spark to spark-orig + Execute(("mv", + format("{oozie_shared_lib}/lib/spark"), + format("{oozie_shared_lib}/lib/spark-orig")), + sudo=True) + + # Create /usr/hdp/{stack_version}/oozie/share/lib/spark + Directory(format("{oozie_shared_lib}/lib/spark"), + owner = oozie_user, + create_parents = True + ) + + # Copy oozie-sharelib-spark from /usr/hdp/{stack_version}/oozie/share/lib/spark-orig to spark + Execute(format("cp -f {oozie_shared_lib}/lib/spark-orig/oozie-sharelib-spark*.jar {oozie_shared_lib}/lib/spark"), + user=oozie_user) + + # Copy /usr/hdp/{stack_version}/spark-client/*.jar except spark-examples*.jar + Execute(format("cp -P {spark_client_dir}/lib/*.jar {oozie_shared_lib}/lib/spark"), + user=oozie_user) + Execute(format("find {oozie_shared_lib}/lib/spark/ -type l -delete"), + user=oozie_user) + try: + Execute(format("rm -f {oozie_shared_lib}/lib/spark/spark-examples*.jar"), + user=oozie_user) + except: + print "No spark-examples jar files found in Spark client lib." + + # Copy /usr/hdp/{stack_version}/spark-client/python/lib/*.zip & *.jar to /usr/hdp/{stack_version}/oozie/share/lib/spark + Execute(format("cp -f {spark_client_dir}/python/lib/*.zip {oozie_shared_lib}/lib/spark"), + user=oozie_user) + + try: + Execute(format("cp -f {spark_client_dir}/python/lib/*.jar {oozie_shared_lib}/lib/spark"), + user=oozie_user) + except: + print "No jar files found in Spark client python lib." + + # Skipping this step since it might cause issues to automated scripts that rely on hdfs://user/oozie/share/lib + # Rename /usr/hdp/{stack_version}/oozie/share/lib to lib_ts + # millis = int(round(time.time() * 1000)) + # Execute(("mv", + # format("{oozie_shared_lib}/lib"), + # format("{oozie_shared_lib}/lib_{millis}")), + # sudo=True) + params.HdfsResource(format("{oozie_hdfs_user_dir}/share"), action="create_on_execute", type = 'directory', @@ -353,7 +401,7 @@ with Environment() as env: createHdfsResources() putSQLDriverToOozieShared() putCreatedHdfsResourcesToIgnore(env) - + # jar shouldn't be used before (read comment below) File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"), mode=0644,