This is an automated email from the ASF dual-hosted git repository. wuzhiguo pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push: new b5ecc35764 AMBARI-25772: Upgrade Zeppelin for BIGTOP to be compatible with bigtop-select (#3455) b5ecc35764 is described below commit b5ecc35764f19292658e45960a450161f4573293 Author: Yu Hou <524860...@qq.com> AuthorDate: Sat Nov 5 17:57:11 2022 +0800 AMBARI-25772: Upgrade Zeppelin for BIGTOP to be compatible with bigtop-select (#3455) --- .../services/ZEPPELIN/package/scripts/master.py | 33 ++++++------ .../services/ZEPPELIN/package/scripts/params.py | 58 ++++++++++------------ 2 files changed, 40 insertions(+), 51 deletions(-) diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ZEPPELIN/package/scripts/master.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ZEPPELIN/package/scripts/master.py index 619d4e082f..10c42be092 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ZEPPELIN/package/scripts/master.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ZEPPELIN/package/scripts/master.py @@ -52,9 +52,6 @@ class Master(Script): if params.spark_version: Execute('echo spark_version:' + str(params.spark_version) + ' detected for spark_home: ' + params.spark_home + ' >> ' + params.zeppelin_log_file, user=params.zeppelin_user) - if params.spark2_version: - Execute('echo spark2_version:' + str(params.spark2_version) + ' detected for spark2_home: ' - + params.spark2_home + ' >> ' + params.zeppelin_log_file, user=params.zeppelin_user) def create_zeppelin_dir(self, params): params.HdfsResource(format("/user/{zeppelin_user}"), @@ -120,7 +117,7 @@ class Master(Script): self.create_zeppelin_log_dir(env) # create the pid and zeppelin dirs - Directory([params.zeppelin_pid_dir, params.zeppelin_dir], + Directory([params.zeppelin_pid_dir, params.zeppelin_home], owner=params.zeppelin_user, group=params.zeppelin_group, cd_access="a", @@ -130,23 +127,23 @@ class Master(Script): self.chown_zeppelin_pid_dir(env) XmlConfig("zeppelin-site.xml", - conf_dir=params.conf_dir, + conf_dir=params.zeppelin_conf_dir, configurations=params.config['configurations']['zeppelin-site'], owner=params.zeppelin_user, group=params.zeppelin_group ) # write out zeppelin-env.sh env_content = InlineTemplate(params.zeppelin_env_content) - File(format("{params.conf_dir}/zeppelin-env.sh"), content=env_content, + File(format("{params.zeppelin_conf_dir}/zeppelin-env.sh"), content=env_content, owner=params.zeppelin_user, group=params.zeppelin_group) # write out shiro.ini shiro_ini_content = InlineTemplate(params.shiro_ini_content) - File(format("{params.conf_dir}/shiro.ini"), content=shiro_ini_content, + File(format("{params.zeppelin_conf_dir}/shiro.ini"), content=shiro_ini_content, owner=params.zeppelin_user, group=params.zeppelin_group) # write out log4j.properties - File(format("{params.conf_dir}/log4j.properties"), content=params.log4j_properties_content, + File(format("{params.zeppelin_conf_dir}/log4j.properties"), content=params.log4j_properties_content, owner=params.zeppelin_user, group=params.zeppelin_group) self.create_zeppelin_hdfs_conf_dir(env) @@ -218,7 +215,7 @@ class Master(Script): import params self.create_zeppelin_log_dir(env) self.chown_zeppelin_pid_dir(env) - Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh stop >> ' + params.zeppelin_log_file, + Execute(params.zeppelin_home + '/bin/zeppelin-daemon.sh stop >> ' + params.zeppelin_log_file, user=params.zeppelin_user) def start(self, env, upgrade_type=None): @@ -254,15 +251,15 @@ class Master(Script): ) # if first_setup: - if not glob.glob(params.conf_dir + "/interpreter.json") and \ - not os.path.exists(params.conf_dir + "/interpreter.json"): + if not glob.glob(params.zeppelin_conf_dir + "/interpreter.json") and \ + not os.path.exists(params.zeppelin_conf_dir + "/interpreter.json"): self.create_interpreter_json() if params.zeppelin_interpreter_config_upgrade == True: self.reset_interpreter_settings(upgrade_type) self.update_zeppelin_interpreter() - Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh restart >> ' + Execute(params.zeppelin_home + '/bin/zeppelin-daemon.sh restart >> ' + params.zeppelin_log_file, user=params.zeppelin_user) pidfile = glob.glob(os.path.join(status_params.zeppelin_pid_dir, 'zeppelin-' + params.zeppelin_user + '*.pid'))[0] @@ -414,7 +411,7 @@ class Master(Script): kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};") else: kinit_if_needed = '' - interpreter_config = os.path.join(params.conf_dir, "interpreter.json") + interpreter_config = os.path.join(params.zeppelin_conf_dir, "interpreter.json") shell.call(format("rm {interpreter_config};" "{kinit_if_needed} hdfs --config {hadoop_conf_dir} dfs -get {zeppelin_conf_fs} {interpreter_config}"), user=params.zeppelin_user) @@ -426,7 +423,7 @@ class Master(Script): import json self.copy_interpreter_from_HDFS_to_FS(params) - interpreter_config = os.path.join(params.conf_dir, "interpreter.json") + interpreter_config = os.path.join(params.zeppelin_conf_dir, "interpreter.json") config_content = sudo.read_file(interpreter_config) config_data = json.loads(config_content) return config_data @@ -435,7 +432,7 @@ class Master(Script): import params import json - interpreter_config = os.path.join(params.conf_dir, "interpreter.json") + interpreter_config = os.path.join(params.zeppelin_conf_dir, "interpreter.json") File(interpreter_config, group=params.zeppelin_group, owner=params.zeppelin_user, @@ -653,7 +650,7 @@ class Master(Script): if not self.copy_interpreter_from_HDFS_to_FS(params): interpreter_json = interpreter_json_template.template - File(format("{params.conf_dir}/interpreter.json"), + File(format("{params.zeppelin_conf_dir}/interpreter.json"), content=interpreter_json, owner=params.zeppelin_user, group=params.zeppelin_group, @@ -663,7 +660,7 @@ class Master(Script): params.HdfsResource(self.get_zeppelin_conf_FS(params), type="file", action="create_on_execute", - source=format("{params.conf_dir}/interpreter.json"), + source=format("{params.zeppelin_conf_dir}/interpreter.json"), owner=params.zeppelin_user, recursive_chown=True, recursive_chmod=True, @@ -671,7 +668,7 @@ class Master(Script): def get_zeppelin_spark_dependencies(self): import params - return glob.glob(params.zeppelin_dir + '/interpreter/spark/dep/zeppelin-spark-dependencies*.jar') + return glob.glob(params.zeppelin_home + '/interpreter/spark/dep/zeppelin-spark-dependencies*.jar') if __name__ == "__main__": Master().execute() diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ZEPPELIN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ZEPPELIN/package/scripts/params.py index 051e3e73aa..89095308d2 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ZEPPELIN/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/ZEPPELIN/package/scripts/params.py @@ -50,16 +50,20 @@ def extract_spark_version(spark_home): # server configurations config = Script.get_config() -# stack_root = Script.get_stack_root() -stack_root = "/usr/lib" +stack_root = Script.get_stack_root() +# e.g. 2.3 +stack_version_unformatted = config['clusterLevelParams']['stack_version'] +# e.g. 2.3.0.0 +stack_version_formatted = format_stack_version(stack_version_unformatted) +major_stack_version = get_major_version(stack_version_formatted) +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +# e.g. 2.3.0.0-2130 +version = default("/commandParams/version", None) +stack_name = default("/clusterLevelParams/stack_name", None) # e.g. /var/lib/ambari-agent/cache/stacks/HDP/2.2/services/zeppelin-stack/package service_packagedir = os.path.realpath(__file__).split('/scripts')[0] -zeppelin_dirname = 'zeppelin-server' - -install_dir = os.path.join(stack_root, "zeppelin") - security_enabled = config['configurations']['cluster-env']['security_enabled'] ui_ssl_enabled = config['configurations']['zeppelin-site']['zeppelin.ssl'] @@ -68,20 +72,6 @@ is_ui_ssl_enabled = str(ui_ssl_enabled).upper() == 'TRUE' setup_view = True temp_file = config['configurations']['zeppelin-env']['zeppelin.temp.file'] -spark_home = config['configurations']['zeppelin-env']['spark_home'] -spark_version = None -spark2_home = "" -spark2_version = None -if 'spark-defaults' in config['configurations']: - spark_home = os.path.join(stack_root, "current", 'spark-client') - spark_version = extract_spark_version(spark_home) -if 'spark2-defaults' in config['configurations']: - spark2_home = os.path.join(stack_root, "current", 'spark2-client') - spark2_version = extract_spark_version(spark2_home) - -# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade -version = default("/commandParams/version", None) -stack_name = default("/clusterLevelParams/stack_name", None) # params from zeppelin-site zeppelin_port = str(config['configurations']['zeppelin-site']['zeppelin.server.port']) @@ -107,9 +97,21 @@ hbase_conf_dir = config['configurations']['zeppelin-env']['hbase_conf_dir'] zeppelin_log_file = os.path.join(zeppelin_log_dir, 'zeppelin-setup.log') zeppelin_hdfs_user_dir = format("/user/{zeppelin_user}") -zeppelin_dir = install_dir -conf_dir = "/etc/zeppelin/conf" -external_dependency_conf = "/etc/zeppelin/conf/external-dependency-conf" +zeppelin_conf_dir = "/etc/zeppelin/conf" +external_dependency_conf = format("{zeppelin_conf_dir}/external-dependency-conf") +zeppelin_home = "/usr/lib/zeppelin" + +spark_home = config['configurations']['zeppelin-env']['spark_home'] + +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + spark_home = format("{stack_root}/current/spark-client") + hbase_home = format("{stack_root}/current/hbase-client") + zeppelin_home = format("{stack_root}/current/zeppelin-server") + local_notebook_dir = format("{stack_root}/{stack_version_formatted}/{local_notebook_dir}") + +spark_version = None +if 'spark-defaults' in config['configurations']: + spark_version = extract_spark_version(spark_home) conf_stored_in_hdfs = False if 'zeppelin.config.fs.dir' in config['configurations']['zeppelin-site'] and \ @@ -244,17 +246,7 @@ else: exclude_interpreter_autoconfig = default("/configurations/zeppelin-site/exclude.interpreter.autoconfig", None) -# e.g. 2.3 -stack_version_unformatted = config['clusterLevelParams']['stack_version'] - -# e.g. 2.3.0.0 -stack_version_formatted = format_stack_version(stack_version_unformatted) -major_stack_version = get_major_version(stack_version_formatted) - -# e.g. 2.3.0.0-2130 -full_stack_version = default("/commandParams/version", None) -spark_client_version = get_stack_version('spark-client') hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) livy_hosts = default("/clusterHostInfo/livy_server_hosts", []) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@ambari.apache.org For additional commands, e-mail: commits-h...@ambari.apache.org