Repository: ambari Updated Branches: refs/heads/trunk 76983f3e7 -> 02ac3940c
AMBARI-9811. Clean up spark stack (Gautam Borad via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/02ac3940 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/02ac3940 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/02ac3940 Branch: refs/heads/trunk Commit: 02ac3940c87cb53e6c9a98ab7479816e257019ec Parents: 76983f3 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Tue Mar 3 09:40:57 2015 -0800 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Tue Mar 3 09:45:10 2015 -0800 ---------------------------------------------------------------------- .../1.2.0.2.2/package/scripts/job_history_server.py | 15 +++------------ .../SPARK/1.2.0.2.2/package/scripts/setup_spark.py | 14 ++++++++------ .../SPARK/1.2.0.2.2/package/scripts/spark_client.py | 2 +- .../resources/stacks/HDP/2.2/role_command_order.json | 1 + 4 files changed, 13 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/02ac3940/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py index 05fd604..518352f 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py @@ -43,15 +43,12 @@ class JobHistoryServer(Script): def install(self, env): self.install_packages(env) import params - env.set_params(params) - self.configure(env) def stop(self, env, rolling_restart=False): import params env.set_params(params) - self.configure(env) daemon_cmd = format('{spark_history_server_stop}') Execute(daemon_cmd, user=params.spark_user, @@ -65,18 +62,12 @@ class JobHistoryServer(Script): import params env.set_params(params) - self.configure(env) + setup_spark(env, 'server', action = 'start') if params.security_enabled: spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ") Execute(spark_kinit_cmd, user=params.spark_user) - # FIXME! TODO! remove this after soft link bug is fixed: - #if not os.path.islink('/usr/hdp/current/spark'): - # hdp_version = get_hdp_version() - # cmd = 'ln -s /usr/hdp/' + hdp_version + '/spark /usr/hdp/current/spark' - # Execute(cmd) - daemon_cmd = format('{spark_history_server_start}') no_op_test = format( 'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1') @@ -94,12 +85,12 @@ class JobHistoryServer(Script): # Recursively check all existing gmetad pid files check_process_status(pid_file) - + # Note: This function is not called from start()/install() def configure(self, env): import params env.set_params(params) - setup_spark(env) + setup_spark(env, 'server', action = 'config') if __name__ == "__main__": JobHistoryServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/02ac3940/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py index 8df5378..2c7d287 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py @@ -28,7 +28,7 @@ from resource_management.core.logger import Logger from resource_management.core import shell -def setup_spark(env): +def setup_spark(env, type, action = None): import params env.set_params(params) @@ -38,11 +38,13 @@ def setup_spark(env): group=params.user_group, recursive=True ) - #params.HdfsDirectory(params.spark_hdfs_user_dir, - # action="create", - # owner=params.spark_user, - # mode=0775 - #) + if type == 'server': + if action == 'start' or action == 'config': + params.HdfsDirectory(params.spark_hdfs_user_dir, + action="create", + owner=params.spark_user, + mode=0775 + ) file_path = params.spark_conf + '/spark-defaults.conf' create_file(file_path) http://git-wip-us.apache.org/repos/asf/ambari/blob/02ac3940/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py index b0cb39d..970dc6d 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py @@ -46,7 +46,7 @@ class SparkClient(Script): import params env.set_params(params) - setup_spark(env) + setup_spark(env, 'client', action = 'config') def status(self, env): raise ClientComponentHasNoStatus() http://git-wip-us.apache.org/repos/asf/ambari/blob/02ac3940/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json index 928caa8..c86e2b1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json @@ -6,6 +6,7 @@ "RANGER_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_ADMIN-START"], "RANGER_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_USERSYNC-START"], "SPARK_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK_JOBHISTORYSERVER-START", "APP_TIMELINE_SERVER-START"], + "SPARK_JOBHISTORYSERVER-START" : ["NAMENODE-START"], "FALCON_SERVER-START": ["NAMENODE-START", "DATANODE-START", "OOZIE_SERVER-START"], "WEBHCAT_SERVICE_CHECK-SERVICE_CHECK": ["WEBHCAT_SERVER-START"], "FLUME_SERVICE_CHECK-SERVICE_CHECK": ["FLUME_HANDLER-START"],