AMBARI-20790. Miscellaneous fixes to support HDP 3.0 Tech Preview (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d5e444fb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d5e444fb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d5e444fb Branch: refs/heads/branch-feature-AMBARI-12556 Commit: d5e444fbb984ef58b5b8f28e354a95d063bfb89d Parents: 6c8458f Author: Alejandro Fernandez <afernan...@hortonworks.com> Authored: Wed Apr 19 19:29:11 2017 -0700 Committer: Alejandro Fernandez <afernan...@hortonworks.com> Committed: Thu Apr 20 10:59:55 2017 -0700 ---------------------------------------------------------------------- .../libraries/script/script.py | 6 +- .../common-services/SPARK/2.2.0/metainfo.xml | 13 +- .../SPARK/2.2.0/scripts/job_history_server.py | 8 +- .../SPARK/2.2.0/scripts/livy_server.py | 8 +- .../SPARK/2.2.0/scripts/params.py | 18 +- .../SPARK/2.2.0/scripts/spark_client.py | 8 +- .../SPARK/2.2.0/scripts/spark_service.py | 9 +- .../SPARK/2.2.0/scripts/spark_thrift_server.py | 8 +- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 36 ++-- .../common-services/YARN/3.0.0.3.0/metainfo.xml | 4 +- .../YARN/3.0.0.3.0/package/scripts/yarn.py | 36 ++-- .../services/HDFS/configuration/hadoop-env.xml | 2 +- .../YARN/configuration-mapred/mapred-env.xml | 2 +- .../YARN/configuration-mapred/mapred-site.xml | 12 +- .../stacks/2.0.6/YARN/test_historyserver.py | 198 +++++++++---------- .../stacks/2.0.6/YARN/test_nodemanager.py | 196 +++++++++--------- .../stacks/2.0.6/YARN/test_resourcemanager.py | 52 ++--- .../stacks/2.1/YARN/test_apptimelineserver.py | 40 ++-- .../test/python/stacks/2.3/YARN/test_ats_1_5.py | 98 ++++----- 19 files changed, 385 insertions(+), 369 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 2c441ad..d0e5678 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -370,7 +370,7 @@ class Script(object): show_logs(log_folder, user, lines_count=COUNT_OF_LAST_LINES_OF_OUT_FILES_LOGGED, mask=OUT_FILES_MASK) - def post_start(self, env): + def post_start(self, env=None): pid_files = self.get_pid_files() if pid_files == []: Logger.logger.warning("Pid files for current script are not defined") @@ -827,7 +827,7 @@ class Script(object): # To remain backward compatible with older stacks, only pass upgrade_type if available. # TODO, remove checking the argspec for "upgrade_type" once all of the services support that optional param. - self.pre_start() + self.pre_start(env) if "upgrade_type" in inspect.getargspec(self.start).args: self.start(env, upgrade_type=upgrade_type) else: @@ -835,7 +835,7 @@ class Script(object): self.start(env, rolling_restart=(upgrade_type == UPGRADE_TYPE_ROLLING)) else: self.start(env) - self.post_start() + self.post_start(env) if is_stack_upgrade: # Remain backward compatible with the rest of the services that haven't switched to using http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/common-services/SPARK/2.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/metainfo.xml index 25a6855..d2e2d8b 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/metainfo.xml @@ -240,18 +240,19 @@ <service>HIVE</service> </requiredServices> + <!-- TODO, change these to "spark" and "livy" after RPM switches the name. --> <osSpecifics> <osSpecific> <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> <packages> <package> - <name>spark_${stack_version}</name> + <name>spark2_${stack_version}</name> </package> <package> - <name>spark_${stack_version}-python</name> + <name>spark2_${stack_version}-python</name> </package> <package> - <name>livy_${stack_version}</name> + <name>livy2_${stack_version}</name> </package> </packages> </osSpecific> @@ -259,13 +260,13 @@ <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> <packages> <package> - <name>spark-${stack_version}</name> + <name>spark2-${stack_version}</name> </package> <package> - <name>spark-${stack_version}-python</name> + <name>spark2-${stack_version}-python</name> </package> <package> - <name>livy-${stack_version}</name> + <name>livy2-${stack_version}</name> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/job_history_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/job_history_server.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/job_history_server.py index 16a2224..3937c88 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/job_history_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/job_history_server.py @@ -68,7 +68,8 @@ class JobHistoryServer(Script): def get_component_name(self): - return "spark-historyserver" + # TODO, change to "spark" after RPM switches the name + return "spark2-historyserver" def pre_upgrade_restart(self, env, upgrade_type=None): import params @@ -76,8 +77,9 @@ class JobHistoryServer(Script): env.set_params(params) if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): Logger.info("Executing Spark Job History Server Stack Upgrade pre-restart") - conf_select.select(params.stack_name, "spark", params.version) - stack_select.select("spark-historyserver", params.version) + # TODO, change to "spark" after RPM switches the name + conf_select.select(params.stack_name, "spark2", params.version) + stack_select.select("spark2-historyserver", params.version) # Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not # need to copy the tarball, otherwise, copy it. http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/livy_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/livy_server.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/livy_server.py index b09d9a9..269c97d 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/livy_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/livy_server.py @@ -126,7 +126,8 @@ class LivyServer(Script): Logger.info("DFS directory '" + dir_path + "' exists.") def get_component_name(self): - return "livy-server" + # TODO, change to "livy" after RPM switches the name + return "livy2-server" def pre_upgrade_restart(self, env, upgrade_type=None): import params @@ -134,8 +135,9 @@ class LivyServer(Script): env.set_params(params) if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): Logger.info("Executing Livy Server Stack Upgrade pre-restart") - conf_select.select(params.stack_name, "spark", params.version) - stack_select.select("livy-server", params.version) + # TODO, change to "spark" and "livy" after RPM switches the name + conf_select.select(params.stack_name, "spark2", params.version) + stack_select.select("livy2-server", params.version) def get_log_folder(self): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/params.py index ab58cb6..e60cab5 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/params.py @@ -35,12 +35,13 @@ from resource_management.libraries.script.script import Script # a map of the Ambari role to the component name # for use with <stack-root>/current/<component> +# TODO, change to "spark" and "livy" after RPM switches the name SERVER_ROLE_DIRECTORY_MAP = { - 'SPARK_JOBHISTORYSERVER' : 'spark-historyserver', - 'SPARK_CLIENT' : 'spark-client', - 'SPARK_THRIFTSERVER' : 'spark-thriftserver', - 'LIVY_SERVER' : 'livy-server', - 'LIVY_CLIENT' : 'livy-client' + 'SPARK_JOBHISTORYSERVER' : 'spark2-historyserver', + 'SPARK_CLIENT' : 'spark2-client', + 'SPARK_THRIFTSERVER' : 'spark2-thriftserver', + 'LIVY_SERVER' : 'livy2-server', + 'LIVY_CLIENT' : 'livy2-client' } @@ -59,7 +60,8 @@ sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs() # New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade version = default("/commandParams/version", None) -spark_conf = '/etc/spark/conf' +# TODO, change to "spark" after RPM switches the name +spark_conf = '/etc/spark2/conf' hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hadoop_bin_dir = stack_select.get_hadoop_dir("bin") @@ -139,7 +141,7 @@ has_spark_thriftserver = not len(spark_thriftserver_hosts) == 0 # hive-site params spark_hive_properties = { - 'hive.metastore.uris': config['configurations']['hive-site']['hive.metastore.uris'] + 'hive.metastore.uris': default('/configurations/hive-site/hive.metastore.uris', '') } # security settings @@ -195,7 +197,7 @@ dfs_type = default("/commandParams/dfs_type", "") # livy related config -# livy for spark is only supported from HDP 2.6 +# livy for spark2 is only supported from HDP 2.6 has_livyserver = False if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted): http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_client.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_client.py index a2e53cd..3acde4e 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_client.py +++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_client.py @@ -44,7 +44,8 @@ class SparkClient(Script): raise ClientComponentHasNoStatus() def get_component_name(self): - return "spark-client" + # TODO, change to "spark" after RPM switches the name + return "spark2-client" def pre_upgrade_restart(self, env, upgrade_type=None): import params @@ -52,8 +53,9 @@ class SparkClient(Script): env.set_params(params) if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): Logger.info("Executing Spark Client Stack Upgrade pre-restart") - conf_select.select(params.stack_name, "spark", params.version) - stack_select.select("spark-client", params.version) + # TODO, change to "spark" after RPM switches the name + conf_select.select(params.stack_name, "spark2", params.version) + stack_select.select("spark2-client", params.version) if __name__ == "__main__": SparkClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_service.py index c6619e4..536d798 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_service.py +++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_service.py @@ -41,7 +41,7 @@ def make_tarfile(output_filename, source_dir): parent_dir=os.path.dirname(output_filename) if not os.path.exists(parent_dir): os.makedirs(parent_dir) - os.chmod(parent_dir, 0711) + os.chmod(parent_dir, 0711) with closing(tarfile.open(output_filename, "w:gz")) as tar: for file in os.listdir(source_dir): tar.add(os.path.join(source_dir,file),arcname=file) @@ -58,12 +58,13 @@ def spark_service(name, upgrade_type=None, action=None): effective_version = format_stack_version(effective_version) if name == 'jobhistoryserver' and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): - # create & copy spark-hdp-yarn-archive.tar.gz to hdfs + # TODO, change to "spark" after RPM switches the name + # create & copy spark2-hdp-yarn-archive.tar.gz to hdfs if not params.sysprep_skip_copy_tarballs_hdfs: source_dir=params.spark_home+"/jars" - tmp_archive_file=get_tarball_paths("spark")[1] + tmp_archive_file=get_tarball_paths("spark2")[1] make_tarfile(tmp_archive_file, source_dir) - copy_to_hdfs("spark", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs, replace_existing_files=True) + copy_to_hdfs("spark2", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs, replace_existing_files=True) # create spark history directory params.HdfsResource(params.spark_history_dir, type="directory", http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_thrift_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_thrift_server.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_thrift_server.py index de82c16..8953b35 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_thrift_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_thrift_server.py @@ -63,15 +63,17 @@ class SparkThriftServer(Script): check_process_status(status_params.spark_thrift_server_pid_file) def get_component_name(self): - return "spark-thriftserver" + # TODO, change to "spark" after RPM switches the name + return "spark2-thriftserver" def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) Logger.info("Executing Spark Thrift Server Stack Upgrade pre-restart") - conf_select.select(params.stack_name, "spark", params.version) - stack_select.select("spark-thriftserver", params.version) + # TODO, change to "spark" after RPM switches the name + conf_select.select(params.stack_name, "spark2", params.version) + stack_select.select("spark2-thriftserver", params.version) def get_log_folder(self): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index 52338df..2e1b208 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -45,15 +45,6 @@ def yarn(name=None, config_dir=None): """ import params - if name == 'resourcemanager': - setup_resourcemanager() - elif name == 'nodemanager': - setup_nodemanager() - elif name == 'apptimelineserver': - setup_ats() - elif name == 'historyserver': - setup_historyserver() - if config_dir is None: config_dir = params.hadoop_conf_dir @@ -61,32 +52,41 @@ def yarn(name=None, config_dir=None): Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), owner=params.yarn_user, group=params.user_group, - create_parents = True, + create_parents=True, mode=0755, - cd_access = 'a', + cd_access='a', ) Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir], owner=params.yarn_user, group=params.user_group, - create_parents = True, - cd_access = 'a', + create_parents=True, + cd_access='a', ) - Directory([params.mapred_pid_dir_prefix, params.mapred_pid_dir, params.mapred_log_dir_prefix, params.mapred_log_dir], owner=params.mapred_user, group=params.user_group, - create_parents = True, - cd_access = 'a', + create_parents=True, + cd_access='a', ) Directory([params.yarn_log_dir_prefix], owner=params.yarn_user, group=params.user_group, - create_parents = True, + create_parents=True, ignore_failures=True, - cd_access = 'a', + cd_access='a', ) + # Some of these function calls depend on the directories above being created first. + if name == 'resourcemanager': + setup_resourcemanager() + elif name == 'nodemanager': + setup_nodemanager() + elif name == 'apptimelineserver': + setup_ats() + elif name == 'historyserver': + setup_historyserver() + XmlConfig("core-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['core-site'], http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/metainfo.xml index 18186bd..a802795 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/metainfo.xml @@ -267,8 +267,7 @@ <enabled>true</enabled> <co-locate>YARN/RESOURCEMANAGER</co-locate> </auto-deploy> - - <!-- TODO HDP 3.0, add later after UI is fixed, + <dependencies> <dependency> <name>HDFS/HDFS_CLIENT</name> @@ -292,7 +291,6 @@ </auto-deploy> </dependency> </dependencies> - --> <commandScript> <script>scripts/historyserver.py</script> http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py index d601f8f..c6b4c18 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py @@ -45,15 +45,6 @@ def yarn(name=None, config_dir=None): """ import params - if name == 'resourcemanager': - setup_resourcemanager() - elif name == 'nodemanager': - setup_nodemanager() - elif name == 'apptimelineserver': - setup_ats() - elif name == 'historyserver': - setup_historyserver() - if config_dir is None: config_dir = params.hadoop_conf_dir @@ -61,32 +52,42 @@ def yarn(name=None, config_dir=None): Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), owner=params.yarn_user, group=params.user_group, - create_parents = True, + create_parents=True, mode=0755, - cd_access = 'a', + cd_access='a', ) Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir], owner=params.yarn_user, group=params.user_group, - create_parents = True, - cd_access = 'a', + create_parents=True, + cd_access='a', ) Directory([params.mapred_pid_dir_prefix, params.mapred_pid_dir, params.mapred_log_dir_prefix, params.mapred_log_dir], owner=params.mapred_user, group=params.user_group, - create_parents = True, - cd_access = 'a', + create_parents=True, + cd_access='a', ) Directory([params.yarn_log_dir_prefix], owner=params.yarn_user, group=params.user_group, - create_parents = True, + create_parents=True, ignore_failures=True, - cd_access = 'a', + cd_access='a', ) + # Some of these function calls depend on the directories above being created first. + if name == 'resourcemanager': + setup_resourcemanager() + elif name == 'nodemanager': + setup_nodemanager() + elif name == 'apptimelineserver': + setup_ats() + elif name == 'historyserver': + setup_historyserver() + XmlConfig("core-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['core-site'], @@ -377,6 +378,7 @@ def setup_resourcemanager(): owner=params.yarn_user, group=params.user_group ) + # This depends on the parent directory already existing. File(params.yarn_job_summary_log, owner=params.yarn_user, group=params.user_group http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml index fbda35a..9d504db 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml @@ -165,6 +165,6 @@ <value-attributes> <type>content</type> </value-attributes> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-env.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-env.xml index deb4ef7..a143660 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-env.xml @@ -44,6 +44,6 @@ <value-attributes> <type>content</type> </value-attributes> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml index cbeb08b..6ce4d72 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml @@ -25,7 +25,7 @@ CLASSPATH for MR applications. A comma-separated list of CLASSPATH entries. </description> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> <!-- These configs were inherited from HDP 2.2 --> @@ -37,13 +37,13 @@ This is not an additive property. You must preserve the original value if you want your map and reduce tasks to have access to native libraries (compression, etc) </description> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> <property> <name>mapreduce.application.framework.path</name> <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value> <description/> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> <property> <name>yarn.app.mapreduce.am.admin-command-opts</name> @@ -61,18 +61,18 @@ of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and mapreduce.reduce.env config settings. </description> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> <property> <name>mapreduce.admin.map.child.java.opts</name> <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> <description/> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> <property> <name>mapreduce.admin.reduce.child.java.opts</name> <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> <description/> - <on-ambari-upgrade add="true"/> + <on-ambari-upgrade add="false"/> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py index 7a0514a..fc7fcd8 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py @@ -197,7 +197,55 @@ class TestHistoryServer(RMFTestCase): self.assertNoMoreResources() def assert_configure_default(self): - + self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + ignore_failures = True, + cd_access = 'a', + ) self.assertResourceCalled('HdfsResource', '/app-logs', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, @@ -309,55 +357,6 @@ class TestHistoryServer(RMFTestCase): cd_access = 'a', recursive_ownership = True, ) - self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - ignore_failures = True, - cd_access = 'a', - ) self.assertResourceCalled('XmlConfig', 'core-site.xml', owner = 'hdfs', group = 'hadoop', @@ -464,6 +463,56 @@ class TestHistoryServer(RMFTestCase): ) def assert_configure_secured(self): + self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + ignore_failures = True, + cd_access = 'a', + ) + self.assertResourceCalled('HdfsResource', '/app-logs', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = True, @@ -575,55 +624,6 @@ class TestHistoryServer(RMFTestCase): cd_access = 'a', recursive_ownership = True, ) - self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - ignore_failures = True, - cd_access = 'a', - ) self.assertResourceCalled('XmlConfig', 'core-site.xml', owner = 'hdfs', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py index ab5e2cd..fbc1b85 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py @@ -133,6 +133,56 @@ class TestNodeManager(RMFTestCase): self.assertNoMoreResources() def assert_configure_default(self): + self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + ignore_failures = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/lib/ambari-agent/data/yarn', create_parents = True, mode = 0755, @@ -185,55 +235,7 @@ class TestNodeManager(RMFTestCase): group = 'hadoop', mode = 0644, ) - self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - ignore_failures = True, - cd_access = 'a', - ) + self.assertResourceCalled('XmlConfig', 'core-site.xml', owner = 'hdfs', group = 'hadoop', @@ -340,53 +342,6 @@ class TestNodeManager(RMFTestCase): ) def assert_configure_secured(self): - self.assertResourceCalled('Directory', '/hadoop/yarn/local', - action = ['delete'] - ) - self.assertResourceCalled('Directory', '/hadoop/yarn/log', - action = ['delete'] - ) - self.assertResourceCalled('Directory', '/var/lib/hadoop-yarn',) - self.assertResourceCalled('File', '/var/lib/hadoop-yarn/nm_security_enabled', - content= 'Marker file to track first start after enabling/disabling security. During first start yarn local, log dirs are removed and recreated' - ) - self.assertResourceCalled('Directory', '/var/lib/ambari-agent/data/yarn', - create_parents = True, - mode = 0755, - ) - self.assertResourceCalled('Directory', '/hadoop/yarn/log', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - ignore_failures = True, - mode = 0775, - cd_access='a', - ) - self.assertResourceCalled('File', '/var/lib/ambari-agent/data/yarn/yarn_log_dir_mount.hist', - content = '\n# This file keeps track of the last known mount-point for each dir.\n# It is safe to delete, since it will get regenerated the next time that the component of the service starts.\n# However, it is not advised to delete this file since Ambari may\n# re-create a dir that used to be mounted on a drive but is now mounted on the root.\n# Comments begin with a hash (#) symbol\n# dir,mount_point\n', - owner = 'hdfs', - group = 'hadoop', - mode = 0644, - ) - self.assertResourceCalled('Directory', '/var/lib/ambari-agent/data/yarn', - create_parents = True, - mode = 0755, - ) - self.assertResourceCalled('Directory', '/hadoop/yarn/local', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - ignore_failures = True, - mode = 0755, - cd_access='a', - recursive_mode_flags = {'d': 'a+rwx', 'f': 'a+rw'}, - ) - self.assertResourceCalled('File', '/var/lib/ambari-agent/data/yarn/yarn_local_dir_mount.hist', - content = '\n# This file keeps track of the last known mount-point for each dir.\n# It is safe to delete, since it will get regenerated the next time that the component of the service starts.\n# However, it is not advised to delete this file since Ambari may\n# re-create a dir that used to be mounted on a drive but is now mounted on the root.\n# Comments begin with a hash (#) symbol\n# dir,mount_point\n', - owner = 'hdfs', - group = 'hadoop', - mode = 0644, - ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -436,6 +391,55 @@ class TestNodeManager(RMFTestCase): ignore_failures = True, cd_access = 'a', ) + + self.assertResourceCalled('Directory', '/hadoop/yarn/local', + action = ['delete'] + ) + self.assertResourceCalled('Directory', '/hadoop/yarn/log', + action = ['delete'] + ) + self.assertResourceCalled('Directory', '/var/lib/hadoop-yarn',) + self.assertResourceCalled('File', '/var/lib/hadoop-yarn/nm_security_enabled', + content= 'Marker file to track first start after enabling/disabling security. During first start yarn local, log dirs are removed and recreated' + ) + self.assertResourceCalled('Directory', '/var/lib/ambari-agent/data/yarn', + create_parents = True, + mode = 0755, + ) + self.assertResourceCalled('Directory', '/hadoop/yarn/log', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + ignore_failures = True, + mode = 0775, + cd_access='a', + ) + self.assertResourceCalled('File', '/var/lib/ambari-agent/data/yarn/yarn_log_dir_mount.hist', + content = '\n# This file keeps track of the last known mount-point for each dir.\n# It is safe to delete, since it will get regenerated the next time that the component of the service starts.\n# However, it is not advised to delete this file since Ambari may\n# re-create a dir that used to be mounted on a drive but is now mounted on the root.\n# Comments begin with a hash (#) symbol\n# dir,mount_point\n', + owner = 'hdfs', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('Directory', '/var/lib/ambari-agent/data/yarn', + create_parents = True, + mode = 0755, + ) + self.assertResourceCalled('Directory', '/hadoop/yarn/local', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + ignore_failures = True, + mode = 0755, + cd_access='a', + recursive_mode_flags = {'d': 'a+rwx', 'f': 'a+rw'}, + ) + self.assertResourceCalled('File', '/var/lib/ambari-agent/data/yarn/yarn_local_dir_mount.hist', + content = '\n# This file keeps track of the last known mount-point for each dir.\n# It is safe to delete, since it will get regenerated the next time that the component of the service starts.\n# However, it is not advised to delete this file since Ambari may\n# re-create a dir that used to be mounted on a drive but is now mounted on the root.\n# Comments begin with a hash (#) symbol\n# dir,mount_point\n', + owner = 'hdfs', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('XmlConfig', 'core-site.xml', owner = 'hdfs', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index b7f90c8..a63caeb 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -178,19 +178,6 @@ class TestResourceManager(RMFTestCase): self.assertNoMoreResources() def assert_configure_default(self): - self.assertResourceCalled('Directory', '/etc/hadoop/conf', - mode = 0755, - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', - owner = 'yarn', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -240,6 +227,19 @@ class TestResourceManager(RMFTestCase): ignore_failures = True, cd_access = 'a', ) + self.assertResourceCalled('Directory', '/etc/hadoop/conf', + mode = 0755, + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('XmlConfig', 'core-site.xml', owner = 'hdfs', group = 'hadoop', @@ -346,19 +346,6 @@ class TestResourceManager(RMFTestCase): ) def assert_configure_secured(self): - self.assertResourceCalled('Directory', '/etc/hadoop/conf', - mode = 0755, - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', - owner = 'yarn', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -408,6 +395,19 @@ class TestResourceManager(RMFTestCase): ignore_failures = True, cd_access = 'a', ) + self.assertResourceCalled('Directory', '/etc/hadoop/conf', + mode = 0755, + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('XmlConfig', 'core-site.xml', owner = 'hdfs', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py index 40db813..5730783 100644 --- a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py +++ b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py @@ -92,26 +92,6 @@ class TestAppTimelineServer(RMFTestCase): self.assertNoMoreResources() def assert_configure_default(self): - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access='a' - ) - self.assertResourceCalled('HdfsResource', None, - immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, - security_enabled = False, - hadoop_bin_dir = '/usr/bin', - keytab = UnknownConfigurationMock(), - default_fs = 'hdfs://c6401.ambari.apache.org:8020', - dfs_type = '', - hdfs_site = self.getConfig()['configurations']['hdfs-site'], - kinit_path_local = '/usr/bin/kinit', - principal_name = UnknownConfigurationMock(), - user = 'hdfs', - action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', - hadoop_conf_dir = '/etc/hadoop/conf', - ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -161,6 +141,26 @@ class TestAppTimelineServer(RMFTestCase): ignore_failures = True, cd_access = 'a', ) + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access='a' + ) + self.assertResourceCalled('HdfsResource', None, + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + dfs_type = '', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + hadoop_conf_dir = '/etc/hadoop/conf', + ) self.assertResourceCalled('XmlConfig', 'core-site.xml', owner = 'hdfs', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e444fb/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py b/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py index b523412..b46a0ed 100644 --- a/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py +++ b/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py @@ -50,6 +50,55 @@ class TestAts(RMFTestCase): self.assertNoMoreResources() def assert_configure_default(self): + self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred', + owner = 'mapred', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn', + owner = 'yarn', + group = 'hadoop', + ignore_failures = True, + create_parents = True, + cd_access = 'a', + ) self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline', owner = 'yarn', group = 'hadoop', @@ -144,55 +193,6 @@ class TestAts(RMFTestCase): action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hadoop_conf_dir = '/etc/hadoop/conf', ) - self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred', - owner = 'mapred', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn', - owner = 'yarn', - group = 'hadoop', - ignore_failures = True, - create_parents = True, - cd_access = 'a', - ) self.assertResourceCalled('XmlConfig', 'core-site.xml', group = 'hadoop', conf_dir = '/etc/hadoop/conf',