http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/metainfo.xml index d818074..fca2bba 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/metainfo.xml @@ -123,11 +123,6 @@ <name>zip</name> </package> <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_mysql_connector</condition> - </package> - <package> <name>extjs</name> </package> </packages>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/metainfo.xml index d351cbe..47b3705 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/metainfo.xml @@ -126,11 +126,6 @@ <name>zip</name> </package> <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_mysql_connector</condition> - </package> - <package> <name>extjs</name> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py index 0771e93..96f1f1a 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py @@ -35,6 +35,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f from resource_management.libraries.functions.oozie_prepare_war import prepare_war from resource_management.libraries.functions.copy_tarball import get_current_version from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.libraries.script.script import Script from resource_management.libraries.functions.security_commons import update_credential_provider_path from resource_management.core.resources.packaging import Package @@ -189,6 +190,11 @@ def oozie(is_server=False): oozie_ownership() + if params.lzo_enabled: + install_lzo_if_needed() + Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), + ) + if is_server: oozie_server_specific() @@ -275,15 +281,6 @@ def oozie_server_specific(): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) - if params.lzo_enabled: - all_lzo_packages = get_lzo_packages(params.stack_version_unformatted) - Package(all_lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) - Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), - not_if = no_op_test, - ) - prepare_war(params) File(hashcode_file, http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py index 70b89b7..1ae86e8 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py @@ -28,6 +28,7 @@ from resource_management.libraries.functions import get_port_from_url from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.lzo_utils import should_install_lzo from resource_management.libraries.functions.expect import expect from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions.get_architecture import get_architecture @@ -366,6 +367,4 @@ HdfsResource = functools.partial( is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'] -# The logic for LZO also exists in HDFS' params.py -io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() +lzo_enabled = should_install_lzo() http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-site.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-site.xml index 59a6952..5dcdd82 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-site.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-site.xml @@ -65,19 +65,6 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>hadoop.kms.authentication.kerberos.keytab</name> - <value>${user.home}/kms.keytab</value> - <description>Path to the keytab with credentials for the configured Kerberos principal.</description> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>hadoop.kms.authentication.kerberos.principal</name> - <value>HTTP/localhost</value> - <description>The Kerberos principal to use for the HTTP endpoint. The principal must start with 'HTTP/' as per the Kerberos HTTP SPNEGO specification.</description> - <property-type>KERBEROS_PRINCIPAL</property-type> - <on-ambari-upgrade add="true"/> - </property> - <property> <name>hadoop.kms.authentication.kerberos.name.rules</name> <value>DEFAULT</value> <description>Rules used to resolve Kerberos principal names.</description> @@ -112,19 +99,6 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab</name> - <value>/etc/hadoop/conf/kms.keytab</value> - <description>The absolute path for the Kerberos keytab with the credentials to connect to Zookeeper.</description> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal</name> - <value>kms/#HOSTNAME#</value> - <description>The Kerberos service principal used to connect to Zookeeper.</description> - <property-type>KERBEROS_PRINCIPAL</property-type> - <on-ambari-upgrade add="true"/> - </property> - <property> <name>hadoop.kms.security.authorization.manager</name> <value>org.apache.ranger.authorization.kms.authorizer.RangerKmsAuthorizer</value> <description/> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/configuration/kms-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/configuration/kms-site.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/configuration/kms-site.xml index 1e6f7b5..19ce76f 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/configuration/kms-site.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/configuration/kms-site.xml @@ -65,19 +65,6 @@ <on-ambari-upgrade add="false"/> </property> <property> - <name>hadoop.kms.authentication.kerberos.keytab</name> - <value>${user.home}/kms.keytab</value> - <description>Path to the keytab with credentials for the configured Kerberos principal.</description> - <on-ambari-upgrade add="false"/> - </property> - <property> - <name>hadoop.kms.authentication.kerberos.principal</name> - <value>HTTP/localhost</value> - <description>The Kerberos principal to use for the HTTP endpoint. The principal must start with 'HTTP/' as per the Kerberos HTTP SPNEGO specification.</description> - <property-type>KERBEROS_PRINCIPAL</property-type> - <on-ambari-upgrade add="false"/> - </property> - <property> <name>hadoop.kms.authentication.kerberos.name.rules</name> <value>DEFAULT</value> <description>Rules used to resolve Kerberos principal names.</description> @@ -112,19 +99,6 @@ <on-ambari-upgrade add="false"/> </property> <property> - <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab</name> - <value>/etc/hadoop/conf/kms.keytab</value> - <description>The absolute path for the Kerberos keytab with the credentials to connect to Zookeeper.</description> - <on-ambari-upgrade add="false"/> - </property> - <property> - <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal</name> - <value>kms/#HOSTNAME#</value> - <description>The Kerberos service principal used to connect to Zookeeper.</description> - <property-type>KERBEROS_PRINCIPAL</property-type> - <on-ambari-upgrade add="false"/> - </property> - <property> <name>hadoop.kms.security.authorization.manager</name> <value>org.apache.ranger.authorization.kms.authorizer.RangerKmsAuthorizer</value> <description/> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/alerts/alert_spark_thrift_port.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/alerts/alert_spark_thrift_port.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/alerts/alert_spark_thrift_port.py index 3f80fd9..6874cb4 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/alerts/alert_spark_thrift_port.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/alerts/alert_spark_thrift_port.py @@ -27,6 +27,7 @@ from resource_management.libraries.script.script import Script from resource_management.libraries.functions import get_kinit_path from resource_management.core.resources import Execute from resource_management.core import global_lock +from resource_management.core.signal_utils import TerminateStrategy stack_root = Script.get_stack_root() @@ -140,7 +141,12 @@ def execute(configurations={}, parameters={}, host_name=None): start_time = time.time() try: - Execute(cmd, user=hiveruser, path=[beeline_cmd], timeout=CHECK_COMMAND_TIMEOUT_DEFAULT) + Execute(cmd, + user=hiveruser, + path=[beeline_cmd], + timeout=CHECK_COMMAND_TIMEOUT_DEFAULT, + timeout_kill_strategy=TerminateStrategy.KILL_PROCESS_TREE, + ) total_time = time.time() - start_time result_code = 'OK' label = OK_MESSAGE.format(total_time, port) http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py index 53c8f9e..6a29efb 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py @@ -52,7 +52,8 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): owner=params.spark_user, group=params.user_group, mode=0775, - create_parents = True + create_parents = True, + cd_access = 'a', ) if type == 'server' and action == 'config': params.HdfsResource(params.spark_hdfs_user_dir, http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/alerts/alert_spark2_thrift_port.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/alerts/alert_spark2_thrift_port.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/alerts/alert_spark2_thrift_port.py index 9e1afea..d3660de 100644 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/alerts/alert_spark2_thrift_port.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/alerts/alert_spark2_thrift_port.py @@ -27,6 +27,7 @@ from resource_management.libraries.script.script import Script from resource_management.libraries.functions import get_kinit_path from resource_management.core.resources import Execute from resource_management.core import global_lock +from resource_management.core.signal_utils import TerminateStrategy stack_root = Script.get_stack_root() @@ -141,7 +142,12 @@ def execute(configurations={}, parameters={}, host_name=None): start_time = time.time() try: - Execute(cmd, user=hiveruser, path=[beeline_cmd], timeout=CHECK_COMMAND_TIMEOUT_DEFAULT) + Execute(cmd, + user=hiveruser, + path=[beeline_cmd], + timeout=CHECK_COMMAND_TIMEOUT_DEFAULT, + timeout_kill_strategy=TerminateStrategy.KILL_PROCESS_TREE + ) total_time = time.time() - start_time result_code = 'OK' label = OK_MESSAGE.format(total_time, port) http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py index a7b8459..792b2a1 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py @@ -42,7 +42,8 @@ def setup_spark(env, type, upgrade_type = None, action = None): owner=params.spark_user, group=params.user_group, mode=0775, - create_parents = True + create_parents = True, + cd_access = 'a', ) if type == 'server' and action == 'config': params.HdfsResource(params.spark_hdfs_user_dir, http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml index a7ec1ec..56071ec 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml @@ -74,11 +74,6 @@ <package> <name>sqoop</name> </package> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_mysql_connector</condition> - </package> </packages> </osSpecific> </osSpecifics> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py index 400c87c..eaf1ee4 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py @@ -23,11 +23,11 @@ from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_kinit_path import get_kinit_path from resource_management.libraries.script import Script +from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.format import format from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.expect import expect -from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.core.exceptions import Fail @@ -71,7 +71,7 @@ zoo_conf_dir = "/etc/zookeeper" if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): sqoop_conf_dir = format("{stack_root}/current/sqoop-client/conf") sqoop_lib = format("{stack_root}/current/sqoop-client/lib") - hadoop_home = format("{stack_root}/current/hadoop-client") + hadoop_home = stack_select.get_hadoop_dir("home") hbase_home = format("{stack_root}/current/hbase-client") hive_home = format("{stack_root}/current/hive-client") sqoop_bin_dir = format("{stack_root}/current/sqoop-client/bin/") http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml index 999d93a..e02d9f4 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml @@ -69,16 +69,6 @@ </components> <osSpecifics> <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_mysql_connector</condition> - </package> - </packages> - </osSpecific> - <osSpecific> <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> <packages> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py index 400c87c..eaf1ee4 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py @@ -23,11 +23,11 @@ from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_kinit_path import get_kinit_path from resource_management.libraries.script import Script +from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.format import format from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.expect import expect -from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.core.exceptions import Fail @@ -71,7 +71,7 @@ zoo_conf_dir = "/etc/zookeeper" if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): sqoop_conf_dir = format("{stack_root}/current/sqoop-client/conf") sqoop_lib = format("{stack_root}/current/sqoop-client/lib") - hadoop_home = format("{stack_root}/current/hadoop-client") + hadoop_home = stack_select.get_hadoop_dir("home") hbase_home = format("{stack_root}/current/hbase-client") hive_home = format("{stack_root}/current/hive-client") sqoop_bin_dir = format("{stack_root}/current/sqoop-client/bin/") http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py index 4d63685..cef709b 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py @@ -50,7 +50,7 @@ version = default("/commandParams/version", None) hadoop_lib_home = stack_select.get_hadoop_dir("lib") # default hadoop parameters -hadoop_home = '/usr' +hadoop_home = stack_select.get_hadoop_dir("home") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() tez_etc_dir = "/etc/tez" http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py index dfa6501..35647e4 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py @@ -23,8 +23,8 @@ import os # Local Imports from resource_management.core.resources.system import Directory, File +from resource_management.libraries.functions import lzo_utils from resource_management.libraries.resources.xml_config import XmlConfig -from resource_management.libraries.functions.format import format from resource_management.core.source import InlineTemplate from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -37,6 +37,9 @@ def tez(config_dir): """ import params + # ensure that matching LZO libraries are installed for Tez + lzo_utils.install_lzo_if_needed() + Directory(params.tez_etc_dir, mode=0755) Directory(config_dir, http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml index 5c17044..ac5683e 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/configuration/tez-site.xml @@ -78,7 +78,7 @@ </property> <property> <name>tez.am.launch.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if you want to have access to native libraries. @@ -124,7 +124,7 @@ </property> <property> <name>tez.task.launch.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if you want to have access to native libraries. http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_linux.py index 5a028bd..2b3fa38 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/params_linux.py @@ -48,7 +48,7 @@ stack_version_formatted = format_stack_version(stack_version_unformatted) version = default("/commandParams/version", None) # default hadoop parameters -hadoop_home = '/usr' +hadoop_home = stack_select.get_hadoop_dir("home") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() tez_etc_dir = "/etc/tez" http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez.py b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez.py index dfa6501..f819940 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/package/scripts/tez.py @@ -23,6 +23,7 @@ import os # Local Imports from resource_management.core.resources.system import Directory, File +from resource_management.libraries.functions import lzo_utils from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.libraries.functions.format import format from resource_management.core.source import InlineTemplate @@ -37,6 +38,9 @@ def tez(config_dir): """ import params + # ensure that matching LZO libraries are installed for Tez + lzo_utils.install_lzo_if_needed() + Directory(params.tez_etc_dir, mode=0755) Directory(config_dir, http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml index a7d8cd6..398c9d7 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml @@ -438,7 +438,7 @@ </property> <property> <name>mapreduce.admin.user.env</name> - <value>LD_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:./mr-framework/hadoop/lib/native/Linux-{{architecture}}-64:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for map and reduce task processes. This is not an additive property. You must preserve the original value if http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 620408b..3e4504d 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -65,7 +65,7 @@ stack_name = status_params.stack_name stack_root = Script.get_stack_root() tarball_map = default("/configurations/cluster-env/tarball_map", None) -config_path = os.path.join(stack_root, "current/hadoop-client/conf") +config_path = stack_select.get_hadoop_dir("conf") config_dir = os.path.realpath(config_path) # get the correct version to use for checking stack features http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index 363d464..d077c84 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -27,6 +27,7 @@ from resource_management.libraries.script.script import Script from resource_management.core.resources.service import ServiceConfig from resource_management.libraries.functions.format import format from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.core.resources.system import Directory from resource_management.core.resources.system import File from resource_management.libraries.resources.xml_config import XmlConfig @@ -45,6 +46,8 @@ def yarn(name=None, config_dir=None): """ import params + install_lzo_if_needed() + if config_dir is None: config_dir = params.hadoop_conf_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration-mapred/mapred-site.xml index 705763f..3b31db1 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration-mapred/mapred-site.xml @@ -438,7 +438,7 @@ </property> <property> <name>mapreduce.admin.user.env</name> - <value>LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:/usr/lib/hadoop/lib/native/Linux-amd64-64</value> + <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:./mr-framework/hadoop/lib/native/Linux-{{architecture}}-64:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for map and reduce task processes. This is not an additive property. You must preserve the original value if http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py index e4dbe2c..617dc3b 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py @@ -65,7 +65,7 @@ stack_name = status_params.stack_name stack_root = Script.get_stack_root() tarball_map = default("/configurations/cluster-env/tarball_map", None) -config_path = os.path.join(stack_root, "current/hadoop-client/conf") +config_path = stack_select.get_hadoop_dir("conf") config_dir = os.path.realpath(config_path) # get the correct version to use for checking stack features http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py index ef314f3..7731aeb 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py @@ -27,6 +27,7 @@ from resource_management.libraries.script.script import Script from resource_management.core.resources.service import ServiceConfig from resource_management.libraries.functions.format import format from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.core.resources.system import Directory from resource_management.core.resources.system import File from resource_management.libraries.resources.xml_config import XmlConfig @@ -44,6 +45,8 @@ def yarn(name=None, config_dir=None): :param config_dir: Which config directory to write configs to, which could be different during rolling upgrade. """ import params + + install_lzo_if_needed() if config_dir is None: config_dir = params.hadoop_conf_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py index 6ccdfba..5efc277 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py @@ -257,7 +257,6 @@ class Master(Script): if not glob.glob(params.conf_dir + "/interpreter.json") and \ not os.path.exists(params.conf_dir + "/interpreter.json"): self.create_interpreter_json() - self.update_zeppelin_interpreter() if params.zeppelin_interpreter_config_upgrade == True: self.reset_interpreter_settings() @@ -599,6 +598,16 @@ class Master(Script): group=params.zeppelin_group, mode=0664) + if params.conf_stored_in_hdfs: + params.HdfsResource(self.get_zeppelin_conf_FS(params), + type="file", + action="create_on_execute", + source=format("{params.conf_dir}/interpreter.json"), + owner=params.zeppelin_user, + recursive_chown=True, + recursive_chmod=True, + replace_existing_files=True) + def get_zeppelin_spark_dependencies(self): import params return glob.glob(params.zeppelin_dir + '/interpreter/spark/dep/zeppelin-spark-dependencies*.jar') http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index 47f9a74..fff18bb 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -179,8 +179,18 @@ class InstallPackages(Script): and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"): Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select") + + restricted_packages = conf_select.get_restricted_packages() + + if 0 == len(restricted_packages): + Logger.info("There are no restricted conf-select packages for this installation") + else: + Logger.info("Restricting conf-select packages to {0}".format(restricted_packages)) + for package_name, directories in conf_select.get_package_dirs().iteritems(): - conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories) + if 0 == len(restricted_packages) or package_name in restricted_packages: + conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories) + def compute_actual_version(self): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HIVE/metainfo.xml index d14ce67..d9ee0a3 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HIVE/metainfo.xml @@ -233,10 +233,6 @@ <package> <name>postgresql-jdbc</name> </package> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - </package> </packages> </osSpecific> <osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/OOZIE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/OOZIE/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/OOZIE/metainfo.xml index 20c3b32..8382662 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/OOZIE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/OOZIE/metainfo.xml @@ -121,10 +121,6 @@ <package> <name>oozie-client</name> </package> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HIVE/metainfo.xml index 633f3a6..ca92e98 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HIVE/metainfo.xml @@ -57,10 +57,6 @@ <package> <name>webhcat-tar-pig</name> </package> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - </package> </packages> </osSpecific> <osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/OOZIE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/OOZIE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/OOZIE/metainfo.xml index 27d4ba4..150cbde 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/OOZIE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/OOZIE/metainfo.xml @@ -40,10 +40,6 @@ <package> <name>zip</name> </package> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index 86de20d..f79cfe0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -427,6 +427,26 @@ "name": "hadoop_custom_extensions", "description": "Support hadoop custom extensions", "min_version": "2.6.0.0" + }, + { + "name": "registry_remove_rootpath", + "description": "Registry remove root path setting", + "min_version": "2.6.3.0" + }, + { + "name": "registry_allowed_resources_support", + "description": "Registry allowed resources", + "min_version": "2.6.3.0" + }, + { + "name": "registry_rewriteuri_filter_support", + "description": "Registry RewriteUri servlet filter", + "min_version": "2.6.3.0" + }, + { + "name": "sam_storage_core_in_registry", + "description": "Storage core module moved to registry", + "min_version": "2.6.3.0" } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json index 20b12a9..794d2b0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json @@ -994,6 +994,12 @@ "current_dir": "{0}/current/atlas-client/conf" } ], + "beacon": [ + { + "conf_dir": "/etc/beacon/conf", + "current_dir": "{0}/current/beacon-client/conf" + } + ], "druid": [ { "conf_dir": "/etc/druid/conf", @@ -1060,6 +1066,12 @@ "current_dir": "{0}/current/knox-server/conf" } ], + "livy": [ + { + "conf_dir": "/etc/livy/conf", + "current_dir": "{0}/current/livy-client/conf" + } + ], "mahout": [ { "conf_dir": "/etc/mahout/conf", @@ -1175,10 +1187,99 @@ } ] }, + "conf-select-patching": { + "ACCUMULO": { + "packages": ["accumulo"] + }, + "ATLAS": { + "packages": ["atlas"] + }, + "BEACON": { + "packages": ["beacon"] + }, + "DRUID": { + "packages": ["druid", "superset"] + }, + "FALCON": { + "packages": ["falcon"] + }, + "FLUME": { + "packages": ["flume"] + }, + "HBASE": { + "packages": ["hbase"] + }, + "HDFS": { + "packages": [] + }, + "HIVE": { + "packages": ["hive", "hive-hcatalog", "hive2", "tez_hive2"] + }, + "KAFKA": { + "packages": ["kafka"] + }, + "KNOX": { + "packages": ["knox"] + }, + "MAHOUT": { + "packages": ["mahout"] + }, + "MAPREDUCE2": { + "packages": [] + }, + "OOZIE": { + "packages": ["oozie"] + }, + "PIG": { + "packages": ["pig"] + }, + "R4ML": { + "packages": [] + }, + "RANGER": { + "packages": ["ranger-admin", "ranger-usersync", "ranger-tagsync"] + }, + "RANGER_KMS": { + "packages": ["ranger-kms"] + }, + "SLIDER": { + "packages": ["slider"] + }, + "SPARK": { + "packages": ["spark", "livy"] + }, + "SPARK2": { + "packages": ["spark2", "livy"] + }, + "SQOOP": { + "packages": ["sqoop"] + }, + "STORM": { + "packages": ["storm", "storm-slider-client"] + }, + "SYSTEMML": { + "packages": [] + }, + "TEZ": { + "packages": ["tez"] + }, + "TITAN": { + "packages": [] + }, + "YARN": { + "packages": [] + }, + "ZEPPELIN": { + "packages": ["zeppelin"] + }, + "ZOOKEEPER": { + "packages": ["zookeeper"] + } + }, "upgrade-dependencies" : { - "YARN": ["TEZ"], - "TEZ": ["YARN"], - "MAHOUT": ["MAPREDUCE2"] + "HIVE": ["TEZ"], + "MAHOUT": ["MAPREDUCE2"], + "OOZIE": ["MAPREDUCE2"] } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json index 78a31f1..3f576bf 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json @@ -14,6 +14,7 @@ "HIVE_SERVER-RESTART": ["NODEMANAGER-RESTART", "MYSQL_SERVER-RESTART", "ZOOKEEPER_SERVER-RESTART"], "HUE_SERVER-START": ["HIVE_SERVER-START", "HCAT-START", "OOZIE_SERVER-START"], "FLUME_HANDLER-START": ["OOZIE_SERVER-START"], + "NFS_GATEWAY-START": ["NAMENODE-START"], "MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], "OOZIE_SERVICE_CHECK-SERVICE_CHECK": ["OOZIE_SERVER-START", "MAPREDUCE2_SERVICE_CHECK-SERVICE_CHECK"], "HBASE_SERVICE_CHECK-SERVICE_CHECK": ["HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/HIVE/metainfo.xml index 7bad9fc..7ac6a63 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/HIVE/metainfo.xml @@ -57,10 +57,6 @@ <package> <name>webhcat-tar-pig</name> </package> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - </package> </packages> </osSpecific> <osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/OOZIE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/OOZIE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/OOZIE/metainfo.xml index a62ef78..82e74ae 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/OOZIE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/OOZIE/metainfo.xml @@ -40,10 +40,6 @@ <package> <name>zip</name> </package> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml index 1c8f475..ea6ad25 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml @@ -258,7 +258,7 @@ limitations under the License. </property> <property> <name>hive.auto.convert.sortmerge.join</name> - <value>true</value> + <value>false</value> <description>Will the join be automatically converted to a sort-merge join, if the joined tables pass the criteria for sort-merge join. </description> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml index 28dfb37..f3eaa8c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml @@ -55,11 +55,6 @@ <package> <name>webhcat-tar-pig</name> </package> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_mysql_connector</condition> - </package> </packages> </osSpecific> <osSpecific> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml index 8c2ec8b..d7221b9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml @@ -40,19 +40,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> <package> @@ -77,19 +64,6 @@ <name>snappy-devel</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> </packages> @@ -123,15 +97,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>hadooplzo-${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> <name>libhdfs0-${stack_version}</name> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml index 2073832..0944252 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml @@ -696,7 +696,7 @@ limitations under the License. </property> <property> <name>hive.auto.convert.sortmerge.join</name> - <value>true</value> + <value>false</value> <description>Will the join be automatically converted to a sort-merge join, if the joined tables pass the criteria for sort-merge join.</description> <on-ambari-upgrade add="true"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml index 051cad1..49a4e69 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml @@ -58,16 +58,6 @@ <osSpecifics> <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_mysql_connector</condition> - </package> - </packages> - </osSpecific> - <osSpecific> <osFamily>amazon2015,redhat6,redhat7,suse11,suse12</osFamily> <packages> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml index cb84a5d..87afb5c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml @@ -65,11 +65,6 @@ <name>zip</name> </package> <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_mysql_connector</condition> - </package> - <package> <name>extjs</name> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml index 884019c..c0e6b3e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml @@ -23,16 +23,6 @@ <version>1.4.5.2.2</version> <osSpecifics> <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_mysql_connector</condition> - </package> - </packages> - </osSpecific> - <osSpecific> <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> <packages> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml index 1427a6f..5513ab1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml @@ -78,7 +78,7 @@ </property> <property> <name>tez.am.launch.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if you want to have access to native libraries. @@ -124,7 +124,7 @@ </property> <property> <name>tez.task.launch.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if you want to have access to native libraries. http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml index 4ad08ce..099e388 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml @@ -20,7 +20,7 @@ <configuration xmlns:xi="http://www.w3.org/2001/XInclude"; supports_final="true"> <property> <name>mapreduce.admin.user.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> + <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:./mr-framework/hadoop/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value> <description> Additional execution environment entries for map and reduce task processes. This is not an additive property. You must preserve the original value if http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/metainfo.xml index 0e12f09..7340b79 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/metainfo.xml @@ -24,15 +24,6 @@ <osSpecifics> <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - </package> - </packages> - </osSpecific> - <osSpecific> <osFamily>redhat5,redhat6,suse11</osFamily> <packages> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/SQOOP/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/SQOOP/metainfo.xml index 5238914..5b02eea 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/SQOOP/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/SQOOP/metainfo.xml @@ -23,15 +23,6 @@ <version>1.4.6.2.3</version> <osSpecifics> <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - </package> - </packages> - </osSpecific> - <osSpecific> <osFamily>redhat5,redhat6,suse11</osFamily> <packages> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/metainfo.xml index f98b24b..9fa9b90 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/metainfo.xml @@ -17,7 +17,7 @@ --> <metainfo> <versions> - <active>true</active> + <active>false</active> </versions> <extends>2.2</extends> <minJdk>1.7</minJdk> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml index ccf9a4e..86531cc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml @@ -72,19 +72,6 @@ <name>snappy-devel</name> </package> <package> - <name>lzo</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> <package> @@ -109,19 +96,6 @@ <name>snappy-devel</name> </package> <package> - <name>liblzo2-2</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> - <name>hadooplzo_${stack_version}-native</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>hadoop_${stack_version}-libhdfs</name> </package> </packages> @@ -155,10 +129,6 @@ <name>libsnappy-dev</name> </package> <package> - <name>hadooplzo-${stack_version}</name> - <condition>should_install_lzo</condition> - </package> - <package> <name>libhdfs0-${stack_version}</name> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml index accbedd..53b8dd0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml @@ -24,16 +24,6 @@ <osSpecifics> <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_mysql_connector</condition> - </package> - </packages> - </osSpecific> - <osSpecific> <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> <packages> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index 907626d..bb22c1e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -24,6 +24,7 @@ <check>org.apache.ambari.server.checks.RangerAuditDbCheck</check> <check>org.apache.ambari.server.checks.ServicePresenceCheck</check> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <configuration> <!-- Configuration properties for all pre-reqs including required pre-reqs --> <check-properties name="org.apache.ambari.server.checks.HiveDynamicServiceDiscoveryCheck"> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index 567e6e1..faf8cbc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -36,6 +36,7 @@ <check>org.apache.ambari.server.checks.ServicePresenceCheck</check> <check>org.apache.ambari.server.checks.RangerAuditDbCheck</check> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <!-- Specific to HDP 2.5, Storm is not rolling --> <check>org.apache.ambari.server.checks.StormShutdownWarning</check> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml index faf5b76..97824f2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml @@ -24,6 +24,7 @@ <check>org.apache.ambari.server.checks.RangerAuditDbCheck</check> <check>org.apache.ambari.server.checks.ServicePresenceCheck</check> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <configuration> <!-- Configuration properties for all pre-reqs including required pre-reqs --> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index 572a259..9e56d97 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -37,6 +37,7 @@ <check>org.apache.ambari.server.checks.ServicePresenceCheck</check> <check>org.apache.ambari.server.checks.RangerAuditDbCheck</check> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <!-- Specific to HDP 2.5, Storm is not rolling --> <check>org.apache.ambari.server.checks.StormShutdownWarning</check> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml index 79296c3..1066f6c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml @@ -30,7 +30,7 @@ <name>falcon_${stack_version}</name> </package> <package> - <name>atlas-metadata_${stack_version}-hive-plugin</name> + <name>atlas-metadata_${stack_version}-falcon-plugin</name> <condition>should_install_falcon_atlas_hook</condition> </package> </packages> @@ -42,7 +42,7 @@ <name>falcon-${stack_version}</name> </package> <package> - <name>atlas-metadata-${stack_version}-hive-plugin</name> + <name>atlas-metadata-${stack_version}-falcon-plugin</name> <condition>should_install_falcon_atlas_hook</condition> </package> </packages> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml index f2a1161..fb72d98 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml @@ -142,16 +142,6 @@ </requiredServices> <osSpecifics> <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>mysql-connector-java</name> - <skipUpgrade>true</skipUpgrade> - <condition>should_install_mysql_connector</condition> - </package> - </packages> - </osSpecific> - <osSpecific> <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> <packages> <package> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json index fca14ab..8e285e9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json @@ -102,21 +102,11 @@ }, { "name": "llap_zk_hive", + "reference": "/HIVE/HIVE_SERVER/hive_server_hive", "principal": { - "value": "hive/_HOST@${realm}", - "type" : "service", "configuration": "hive-interactive-site/hive.llap.zk.sm.principal" }, "keytab": { - "file": "${keytab_dir}/hive.llap.zk.sm.keytab", - "owner": { - "name": "${yarn-env/yarn_user}", - "access": "r" - }, - "group": { - "name": "${cluster-env/user_group}", - "access": "r" - }, "configuration": "hive-interactive-site/hive.llap.zk.sm.keytab.file" }, "when" : { http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index b1e5f91..8c2c860 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -104,14 +104,37 @@ </service> <service name="SPARK2"> + <component name="SPARK2_JOBHISTORYSERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j"> + <type>spark2-log4j-properties</type> + <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/> + <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR" replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/> + </definition> + </changes> + </component> <component name="SPARK2_CLIENT"> <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j"> + <type>spark2-log4j-properties</type> + <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/> + <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR" replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/> + </definition> <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue"> <type>spark2-defaults</type> <set key="spark.yarn.queue" value="default" if-type="spark-defaults" if-key="spark.yarn.queue" if-key-state="absent"/> </definition> </changes> </component> + <component name="SPARK2_THRIFTSERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j"> + <type>spark2-log4j-properties</type> + <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/> + <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR" replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/> + </definition> + </changes> + </component> </service> <service name="TEZ"> http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 8012c90..9caa850 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -22,6 +22,7 @@ <type>NON_ROLLING</type> <prerequisite-checks> <check>org.apache.ambari.server.checks.RangerSSLConfigCheck</check> + <check>org.apache.ambari.server.checks.LZOCheck</check> <configuration> <!-- Configuration properties for all pre-reqs including required pre-reqs --> <check-properties name="org.apache.ambari.server.checks.HiveDynamicServiceDiscoveryCheck"> @@ -76,7 +77,7 @@ <service name="FLUME"> <component>FLUME_HANDLER</component> </service> - + <service name="ACCUMULO"> <component>ACCUMULO_TRACER</component> <component>ACCUMULO_GC</component> @@ -354,7 +355,7 @@ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixCapacitySchedulerOrderingPolicy"> <summary>Validate Root Queue Ordering Policy</summary> </task> - </execute-stage> + </execute-stage> <!--TEZ--> <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez"> @@ -678,6 +679,15 @@ <summary>Add queue customization property</summary> </task> </execute-stage> + <execute-stage service="SPARK2" component="SPARK2_THRIFTSERVER" title="Apply config changes for Spark2 ThriftServer log4j"> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j"/> + </execute-stage> + <execute-stage service="SPARK2" component="SPARK2_JOBHISTORYSERVER" title="Apply config changes for Spark2 Job HistoryServer log4j"> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j"/> + </execute-stage> + <execute-stage service="SPARK2" component="SPARK2_CLIENT" title="Apply config changes for Spark2 Client log4j"> + <task xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j"/> + </execute-stage> </group> <!-- @@ -719,7 +729,7 @@ </task> </execute-stage> </group> - + <!-- Now, restart all of the services. --> <group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper"> <service-check>false</service-check> @@ -1082,7 +1092,7 @@ <component>FLUME_HANDLER</component> </service> </group> - + <group xsi:type="restart" name="ACCUMULO" title="Accumulo"> <service-check>false</service-check> <skippable>true</skippable> @@ -1111,7 +1121,7 @@ <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check"> <direction>UPGRADE</direction> - + <execute-stage title="Check Component Versions"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" /> </execute-stage> @@ -1191,7 +1201,7 @@ <function>setup_ranger_java_patches</function> </task> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> <upgrade> @@ -1214,7 +1224,7 @@ <function>configure_atlas_user_for_tagsync</function> </task> </pre-upgrade> - + <pre-downgrade copy-upgrade="true" /> <upgrade>
