Repository: ambari Updated Branches: refs/heads/trunk 98586c0ce -> e7e350390
AMBARI-11483. Ranger: Admin RU does not complete for HDP-2.2 -> HDP-2.3 (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e7e35039 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e7e35039 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e7e35039 Branch: refs/heads/trunk Commit: e7e350390a85f237e961f143c87953c42c29213e Parents: 98586c0 Author: Nate Cole <nc...@hortonworks.com> Authored: Thu May 28 10:45:32 2015 -0400 Committer: Nate Cole <nc...@hortonworks.com> Committed: Thu May 28 15:17:17 2015 -0400 ---------------------------------------------------------------------- .../functions/setup_ranger_plugin_xml.py | 5 +- .../upgrades/RangerConfigCalculation.java | 132 +++++++++++++++ .../0.96.0.2.0/package/scripts/hbase_master.py | 2 +- .../package/scripts/hbase_regionserver.py | 2 +- .../package/scripts/setup_ranger_hbase.py | 13 +- .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 4 +- .../package/scripts/setup_ranger_hdfs.py | 11 +- .../RANGER/0.4.0/package/scripts/params.py | 3 +- .../0.4.0/package/scripts/ranger_admin.py | 7 + .../0.4.0/package/scripts/ranger_service.py | 6 +- .../0.4.0/package/scripts/setup_ranger_xml.py | 68 ++++++-- .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 101 +++++++++++ .../upgrades/RangerConfigCalculationTest.java | 166 +++++++++++++++++++ 13 files changed, 490 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py index e9ec4f5..cfa51da 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -44,7 +44,7 @@ def setup_ranger_plugin(component_select_name, service_name, plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes, component_list, audit_db_is_enabled, credential_file, xa_audit_db_password, ssl_truststore_password, - ssl_keystore_password, api_version=None): + ssl_keystore_password, api_version=None, hdp_version_override = None): File(component_downloaded_custom_connector, content = DownloadSource(component_driver_curl_source) @@ -56,6 +56,9 @@ def setup_ranger_plugin(component_select_name, service_name, ) hdp_version = get_hdp_version(component_select_name) + if hdp_version_override is not None: + hdp_version = hdp_version_override + component_conf_dir = conf_dict if plugin_enabled: http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerConfigCalculation.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerConfigCalculation.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerConfigCalculation.java new file mode 100644 index 0000000..ed41c8a --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerConfigCalculation.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.serveraction.upgrades; + +import java.text.MessageFormat; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.HostRoleStatus; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.serveraction.AbstractServerAction; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; + +import com.google.inject.Inject; + +/** + * Computes Ranger properties. This class is only used when moving from + * HDP-2.2 to HDP-2.3 in that upgrade pack. + */ +public class RangerConfigCalculation extends AbstractServerAction { + private static final String SOURCE_CONFIG_TYPE = "admin-properties"; + + @Inject + private Clusters m_clusters; + + @Override + public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) + throws AmbariException, InterruptedException { + + String clusterName = getExecutionCommand().getClusterName(); + + Cluster cluster = m_clusters.getCluster(clusterName); + + Config sourceConfig = cluster.getDesiredConfigByType(SOURCE_CONFIG_TYPE); + + if (null == sourceConfig) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + MessageFormat.format("Source type {0} not found, skipping", SOURCE_CONFIG_TYPE), ""); + } + + String dbProp = "DB_FLAVOR"; + String dbHostProp = "db_host"; + String dbNameProp = "db_name"; + + StringBuilder stdout = new StringBuilder(); + + String db = sourceConfig.getProperties().get(dbProp); + if (null == db) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + MessageFormat.format("Target database from {0}/{1} not found, skipping", + SOURCE_CONFIG_TYPE, dbProp), ""); + } + + stdout.append(MessageFormat.format("Database type is {0}\n", db)); + + db = db.toLowerCase(); + if (!"mysql".equals(db) && !"oracle".equals(db)) { + stdout.append(MessageFormat.format("Target database {0} is not recognized, skipping", db)); + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", stdout.toString(), ""); + } + + String dbHost = sourceConfig.getProperties().get(dbHostProp); + String dbName = sourceConfig.getProperties().get(dbNameProp); + + stdout.append(MessageFormat.format("Database host: {0}\n", dbHost)); + stdout.append(MessageFormat.format("Database name: {0}\n", dbName)); + + if (null == dbHost) { + stdout.append(MessageFormat.format("Hostname must be set using {0}/{1} , skipping", SOURCE_CONFIG_TYPE, dbHostProp)); + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", stdout.toString(), ""); + } + + String driver = null; + String url = null; + String dialect = null; + + if ("mysql".equals(db)) { + if (null == dbName) { + stdout.append(MessageFormat.format("Target database {0} requires {1} to be set, skipping", db, dbName)); + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", stdout.toString(), ""); + } + driver = "com.mysql.jdbc.Driver"; + url = MessageFormat.format("jdbc:mysql://{0}/{1}", dbHost, dbName); + dialect = "org.eclipse.persistence.platform.database.MySQLPlatform"; + } else if ("oracle".equals(db)) { + driver = "oracle.jdbc.OracleDriver"; + url = MessageFormat.format("jdbc:oracle:thin:@//{0}", dbHost); + dialect = "org.eclipse.persistence.platform.database.OraclePlatform"; + } + + stdout.append(MessageFormat.format("Database driver: {0}\n", driver)); + stdout.append(MessageFormat.format("Database url: {0}\n", url)); + stdout.append(MessageFormat.format("Database dialect: {0}", dialect)); + + Config config = cluster.getDesiredConfigByType("ranger-admin-site"); + Map<String, String> targetValues = config.getProperties(); + targetValues.put("ranger.jpa.jdbc.driver", driver); + targetValues.put("ranger.jpa.jdbc.url", url); + targetValues.put("ranger.jpa.jdbc.dialect", dialect); + + targetValues.put("ranger.jpa.audit.jdbc.driver", driver); + targetValues.put("ranger.jpa.audit.jdbc.url", url); + targetValues.put("ranger.jpa.audit.jdbc.dialect", dialect); + + config.setProperties(targetValues); + config.persist(false); + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", stdout.toString(), ""); + } + + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py index 30198c9..19e639c 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py @@ -80,7 +80,7 @@ class HbaseMasterDefault(HbaseMaster): import params env.set_params(params) self.configure(env) # for security - setup_ranger_hbase() + setup_ranger_hbase(rolling_upgrade=rolling_restart) hbase_service('master', action = 'start') def stop(self, env, rolling_restart=False): http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py index 882b982..3244092 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py @@ -83,7 +83,7 @@ class HbaseRegionServerDefault(HbaseRegionServer): import params env.set_params(params) self.configure(env) # for security - setup_ranger_hbase() + setup_ranger_hbase(rolling_upgrade=rolling_restart) hbase_service( 'regionserver', action = 'start' ) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py index 86049f8..6b4dfaa 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py @@ -19,7 +19,7 @@ limitations under the License. """ from resource_management.core.logger import Logger -def setup_ranger_hbase(): +def setup_ranger_hbase(rolling_upgrade = False): import params if params.has_ranger_admin: @@ -29,6 +29,11 @@ def setup_ranger_hbase(): else: from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin + hdp_version = None + + if rolling_upgrade: + hdp_version = params.version + setup_ranger_plugin('hbase-client', 'hbase', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, @@ -42,7 +47,7 @@ def setup_ranger_hbase(): plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hbase-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hbase-policymgr-ssl'], component_list=['hbase-client', 'hbase-master', 'hbase-regionserver'], audit_db_is_enabled=params.xa_audit_db_is_enabled, credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, - ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password - ) + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, + hdp_version_override = hdp_version) else: - Logger.info('Ranger admin not installed') \ No newline at end of file + Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index 923d9df..5e824d0 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -35,7 +35,7 @@ def namenode(action=None, do_format=True, rolling_restart=False, env=None): #additional namenode) create_name_dirs(params.dfs_name_dir) elif action == "start": - setup_ranger_hdfs() + setup_ranger_hdfs(rolling_upgrade = rolling_restart) import params if do_format: format_namenode() @@ -350,4 +350,4 @@ def bootstrap_standby_namenode(params): Logger.warning('Bootstrap standby namenode failed with %d error code. Will retry' % (code)) except Exception as ex: Logger.error('Bootstrap standby namenode threw an exception. Reason %s' %(str(ex))) - return False \ No newline at end of file + return False http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py index f7ddbed..f5df86f 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py @@ -19,7 +19,7 @@ limitations under the License. """ from resource_management.core.logger import Logger -def setup_ranger_hdfs(): +def setup_ranger_hdfs(rolling_upgrade = False): import params if params.has_ranger_admin: @@ -29,6 +29,11 @@ def setup_ranger_hdfs(): else: from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin + hdp_version = None + + if rolling_upgrade: + hdp_version = params.version + setup_ranger_plugin('hadoop-client', 'hdfs', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java_home, @@ -42,7 +47,7 @@ def setup_ranger_hdfs(): plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hdfs-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hdfs-policymgr-ssl'], component_list=['hadoop-client'], audit_db_is_enabled=params.xa_audit_db_is_enabled, credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, - ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password - ) + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, + hdp_version_override = hdp_version) else: Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py index f5da743..7a6dacf 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py @@ -60,7 +60,8 @@ if stack_is_hdp22_or_further: ranger_ugsync_conf = '/etc/ranger/usersync/conf' if stack_is_hdp23_or_further: - ranger_conf = '/usr/hdp/current/ranger-admin/conf' + ranger_conf = '/usr/hdp/current/ranger-admin/conf' + ranger_ugsync_conf = '/usr/hdp/current/ranger-usersync/conf' usersync_services_file = "/usr/hdp/current/ranger-usersync/ranger-usersync-services.sh" http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py index 2c7a599..a471bb4 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py @@ -58,6 +58,13 @@ class RangerAdmin(Script): def pre_rolling_restart(self, env): import params env.set_params(params) + + if params.xml_configurations_supported: + from setup_ranger_xml import ranger, setup_ranger_db, setup_java_patch + ranger('ranger_admin', rolling_upgrade=True) + setup_ranger_db(rolling_upgrade=True) + setup_java_patch(rolling_upgrade=True) + self.set_ru_rangeradmin_in_progress() upgrade.prestart(env, "ranger-admin") http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py index 6e32dac..8ad2627 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_service.py @@ -27,8 +27,10 @@ def ranger_service(name, action=None): Execute(params.ranger_start, environment={'JAVA_HOME': params.java_home}, user=params.unix_user, not_if=no_op_test) elif name == 'ranger_usersync': no_op_test = format('ps -ef | grep proc_rangerusersync | grep -v grep') - + + if params.stack_is_hdp23_or_further: + Execute(('chown','-R', format('{unix_user}:{unix_group}'), format('{usersync_log_dir}/')), sudo=True) Execute(params.usersync_start, environment={'JAVA_HOME': params.java_home}, not_if=no_op_test, @@ -40,4 +42,4 @@ def ranger_service(name, action=None): environment={'JAVA_HOME': params.java_home}, not_if=no_op_test, sudo=True, - ) \ No newline at end of file + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py index df85da8..d845eb4 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py @@ -30,25 +30,36 @@ from resource_management.libraries.functions.is_empty import is_empty # This file contains functions used for setup/configure of Ranger Admin and Ranger Usersync. # The design is to mimic what is done by the setup.sh script bundled by Ranger component currently. -def ranger(name=None): +def ranger(name=None, rolling_upgrade=False): """ parameter name: name of ranger service component """ if name == 'ranger_admin': - setup_ranger_admin() + setup_ranger_admin(rolling_upgrade=rolling_upgrade) if name == 'ranger_usersync': setup_usersync() -def setup_ranger_admin(): +def setup_ranger_admin(rolling_upgrade=False): import params + ranger_home = params.ranger_home + ranger_conf = params.ranger_conf + + if rolling_upgrade: + ranger_home = format("/usr/hdp/{version}/ranger-admin") + ranger_conf = format("/usr/hdp/{version}/ranger-admin/conf") + File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), ) + cp = format("{check_db_connection_jar}") + cp = cp + os.pathsep + format("{driver_curl_target}") + cp = cp + os.pathsep + format("{ranger_home}/ews/webapp/WEB-INF/lib/*") + db_connection_check_command = format( - "{java_home}/bin/java -cp {check_db_connection_jar}:{driver_curl_target} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}") + "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}") Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10) @@ -57,6 +68,16 @@ def setup_ranger_admin(): only_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes/conf"), sudo=True) + if rolling_upgrade: + src_file = format('{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml') + dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml') + Execute(('cp', '-f', src_file, dst_file), sudo=True) + + src_file = format('{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml') + dst_file = format('{ranger_home}/conf/security-applicationContext.xml') + + Execute(('cp', '-f', src_file, dst_file), sudo=True) + Execute(('chown','-R',format('{unix_user}:{unix_group}'), format('{ranger_home}/')), sudo=True) Execute(('ln','-sf', format('{ranger_home}/ews/ranger-admin-services.sh'),'/usr/bin/ranger-admin'), @@ -65,23 +86,23 @@ def setup_ranger_admin(): sudo=True) XmlConfig("ranger-admin-site.xml", - conf_dir=params.ranger_conf, + conf_dir=ranger_conf, configurations=params.config['configurations']['ranger-admin-site'], configuration_attributes=params.config['configuration_attributes']['ranger-admin-site'], owner=params.unix_user, group=params.unix_group, mode=0644) - Directory(os.path.join(params.ranger_conf,'ranger_jaas'), + Directory(os.path.join(ranger_conf,'ranger_jaas'), mode=0700, owner=params.unix_user, group=params.unix_group, ) - do_keystore_setup() + do_keystore_setup(rolling_upgrade=rolling_upgrade) -def setup_ranger_db(): +def setup_ranger_db(rolling_upgrade=False): import params File(params.downloaded_custom_connector, @@ -98,6 +119,10 @@ def setup_ranger_db(): not_if=format("test -f {driver_curl_target}"), sudo=True) + ranger_home = params.ranger_home + if rolling_upgrade: + ranger_home = format("/usr/hdp/{version}/ranger-admin") + if not os.path.isfile(os.path.join(params.ranger_home, 'ews', 'lib',params.jdbc_jar_name)): Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(params.ranger_home, 'ews', 'lib')), path=["/bin", "/usr/bin/"], @@ -111,29 +136,42 @@ def setup_ranger_db(): if params.create_db_dbuser: Logger.info('Setting up Ranger DB and DB User') dba_setup = format('python {ranger_home}/dba_script.py -q') - Execute(dba_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) + Execute(dba_setup, environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) else: Logger.info('Separate DBA property not set. Assuming Ranger DB and DB User exists!') db_setup = format('python {ranger_home}/db_setup.py') - Execute(db_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) + Execute(db_setup, environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) -def setup_java_patch(): +def setup_java_patch(rolling_upgrade=False): import params + ranger_home = params.ranger_home + if rolling_upgrade: + ranger_home = format("/usr/hdp/{version}/ranger-admin") + setup_java_patch = format('python {ranger_home}/db_setup.py -javapatch') - Execute(setup_java_patch, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) + Execute(setup_java_patch, environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) -def do_keystore_setup(): +def do_keystore_setup(rolling_upgrade=False): import params + ranger_home = params.ranger_home + cred_lib_path = params.cred_lib_path + cred_setup_prefix = params.cred_setup_prefix + + if rolling_upgrade: + ranger_home = format("/usr/hdp/{version}/ranger-admin") + cred_lib_path = os.path.join(ranger_home,"cred","lib","*") + cred_setup_prefix = format('python {ranger_home}/ranger_credential_helper.py -l "{cred_lib_path}"') + if not is_empty(params.ranger_credential_provider_path): jceks_path = params.ranger_credential_provider_path cred_setup = format('{cred_setup_prefix} -f {jceks_path} -k "{ranger_jpa_jdbc_credential_alias}" -v "{ranger_ambari_db_password}" -c 1') - Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) + Execute(cred_setup, environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) File(params.ranger_credential_provider_path, owner = params.unix_user, @@ -144,7 +182,7 @@ def do_keystore_setup(): jceks_path = params.ranger_credential_provider_path cred_setup = format('{cred_setup_prefix} -f {jceks_path} -k "{ranger_jpa_audit_jdbc_credential_alias}" -v "{ranger_ambari_audit_db_password}" -c 1') - Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) + Execute(cred_setup, environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) File(params.ranger_credential_provider_path, owner = params.unix_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index 1cbdd88..30d31c7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -311,6 +311,107 @@ <service name="RANGER"> <component name="RANGER_ADMIN"> + <pre-upgrade> + <task xsi:type="configure"> + <type>ranger-env</type> + <set key="xml_configurations_supported" value="true" /> + </task> + <task xsi:type="configure" summary="Updating Ranger Admin"> + <type>ranger-admin-site</type> + <transfer operation="copy" from-type="ranger-site" from-key="HTTPS_CLIENT_AUTH" to-key="xasecure.policymgr.clientssl.keystore" default-value="" /> + <transfer operation="copy" from-type="ranger-site" from-key="HTTPS_KEYSTORE_FILE" to-key="ranger.https.attrib.keystore.file" default-value="" /> + <transfer operation="copy" from-type="ranger-site" from-key="HTTPS_KEYSTORE_PASS" to-key="ranger.service.https.attrib.keystore.pass" default-value="" mask="true" /> + <transfer operation="copy" from-type="ranger-site" from-key="HTTPS_KEY_ALIAS" to-key="ranger.service.https.attrib.keystore.keyalias" default-value="" /> + <transfer operation="copy" from-type="ranger-site" from-key="HTTPS_SERVICE_PORT" to-key="ranger.service.https.port" default-value="" /> + <transfer operation="copy" from-type="ranger-site" from-key="HTTP_ENABLED" to-key="ranger.service.http.enabled" default-value="" /> + <transfer operation="copy" from-type="ranger-site" from-key="HTTP_SERVICE_PORT" to-key="ranger.service.http.port" default-value="" /> + + <transfer operation="copy" from-type="admin-properties" from-key="authServiceHostName" to-key="ranger.unixauth.service.hostname" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="authServicePort" to-key="ranger.unixauth.service.port" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="authentication_method" to-key="ranger.authentication.method" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="remoteLoginEnabled" to-key="ranger.unixauth.remote.login.enabled" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="xa_ldap_url" to-key="ranger.ldap.url" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="xa_ldap_userDNpattern" to-key="ranger.ldap.user.dnpattern" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="xa_ldap_groupSearchBase" to-key="ranger.ldap.group.searchbase" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="xa_ldap_groupSearchFilter" to-key="ranger.ldap.group.searchfilter" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="xa_ldap_groupRoleAttribute" to-key="ranger.ldap.group.roleattribute" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="xa_ldap_ad_domain" to-key="ranger.ldap.ad.domain" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="xa_ldap_ad_url" to-key="ranger.ldap.ad.url" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="policymgr_external_url" to-key="ranger.externalurl" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="db_user" to-key="ranger.jpa.jdbc.user" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="db_password" to-key="ranger.jpa.jdbc.password" default-value="" mask="true" /> + <transfer operation="copy" from-type="admin-properties" from-key="audit_db_user" to-key="ranger.jpa.audit.jdbc.user" default-value="" /> + <transfer operation="copy" from-type="admin-properties" from-key="audit_db_password" to-key="ranger.jpa.audit.jdbc.password" default-value="" mask="true" /> + + <set key="ranger.audit.solr.urls" value="" /> + <set key="ranger.audit.solr.zookeepers" value="" /> + <set key="ranger.audit.solr.username" value="" /> + <set key="ranger.audit.solr.password" value="" /> + <set key="ranger.audit.solr.authentication-method" value="" /> + <set key="ranger.audit.source.type" value="" /> + </task> + + <task xsi:type="server_action" summary="Calculating Ranger Properties" class="org.apache.ambari.server.serveraction.upgrades.RangerConfigCalculation" /> + <task xsi:type="manual"> + <message>ASDFJASD;LFJKASD;LFJASD;FLKAS</message> + </task> + + <task xsi:type="configure" summary="Updating Ranger Usersync"> + <type>ranger-ugsync-site</type> + <transfer operation="copy" from-type="usersync-properties" from-key="CRED_KEYSTORE_FILENAME" to-key="ranger.usersync.credstore.filename" default-value="/etc/ranger/usersync/ugsync.jceks" /> + <transfer operation="copy" from-type="usersync-properties" from-key="MIN_UNIX_USER_ID_TO_SYNC" to-key="ranger.usersync.unix.minUserId" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_INTERVAL" to-key="ranger.usersync.sleeptimeinmillisbetweensynccycle" default-value="60000" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_BIND_DN" to-key="ranger.usersync.ldap.binddn" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_BIND_PASSWORD" to-key="" default-value="ranger.usersync.ldap.ldapbindpassword" mask="true" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_GROUPNAME_CASE_CONVERSION" to-key="ranger.usersync.ldap.groupname.caseconversion" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_URL" to-key="ranger.usersync.ldap.url" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_USERNAME_CASE_CONVERSION" to-key="ranger.usersync.ldap.username.caseconversion" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_USER_GROUP_NAME_ATTRIBUTE" to-key="ranger.usersync.group.memberattributename" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_USER_NAME_ATTRIBUTE" to-key="ranger.usersync.group.nameattribute" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_USER_OBJECT_CLASS" to-key="ranger.usersync.group.objectclass" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_USER_SEARCH_BASE" to-key="ranger.usersync.group.searchbase" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_USER_SEARCH_FILTER" to-key="ranger.usersync.group.searchfilter" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_LDAP_USER_SEARCH_SCOPE" to-key="ranger.usersync.group.searchscope" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="logdir" to-key="ranger.usersync.logdir" default-value="" /> + <transfer operation="copy" from-type="usersync-properties" from-key="SYNC_SOURCE" to-key="ranger.usersync.source.impl.class" default-value="org.apache.ranger.unixusersync.process.UnixUserGroupBuilder" /> + <transfer operation="copy" from-type="usersync-properties" from-key="POLICY_MGR_URL" to-key="ranger.usersync.policymanager.baseURL" default-value="{{ranger_external_url}}" /> + </task> + + <task xsi:type="configure"> + <type>ranger-site</type> + <transfer operation="delete" delete-key="HTTPS_CLIENT_AUTH" /> + <transfer operation="delete" delete-key="HTTPS_KEYSTORE_FILE" /> + <transfer operation="delete" delete-key="HTTPS_KEYSTORE_PASS" /> + <transfer operation="delete" delete-key="HTTPS_KEY_ALIAS" /> + <transfer operation="delete" delete-key="HTTPS_SERVICE_PORT" /> + <transfer operation="delete" delete-key="HTTP_ENABLED" /> + <transfer operation="delete" delete-key="HTTP_SERVICE_PORT" /> + <transfer operation="delete" delete-key="" /> + </task> + + <task xsi:type="configure"> + <type>usersync-properties</type> + <transfer operation="delete" delete-key="CRED_KEYSTORE_FILENAME" /> + <transfer operation="delete" delete-key="MIN_UNIX_USER_ID_TO_SYNC" /> + <transfer operation="delete" delete-key="SYNC_INTERVAL" /> + <transfer operation="delete" delete-key="SYNC_LDAP_BIND_DN" /> + <transfer operation="delete" delete-key="SYNC_LDAP_BIND_PASSWORD" /> + <transfer operation="delete" delete-key="SYNC_LDAP_GROUPNAME_CASE_CONVERSION" /> + <transfer operation="delete" delete-key="SYNC_LDAP_URL" /> + <transfer operation="delete" delete-key="SYNC_LDAP_USERNAME_CASE_CONVERSION" /> + <transfer operation="delete" delete-key="SYNC_LDAP_USER_GROUP_NAME_ATTRIBUTE" /> + <transfer operation="delete" delete-key="SYNC_LDAP_USER_NAME_ATTRIBUTE" /> + <transfer operation="delete" delete-key="SYNC_LDAP_USER_OBJECT_CLASS" /> + <transfer operation="delete" delete-key="SYNC_LDAP_USER_SEARCH_BASE" /> + <transfer operation="delete" delete-key="SYNC_LDAP_USER_SEARCH_FILTER" /> + <transfer operation="delete" delete-key="SYNC_LDAP_USER_SEARCH_SCOPE" /> + <transfer operation="delete" delete-key="logdir" /> + <transfer operation="delete" delete-key="SYNC_SOURCE" /> + <transfer operation="delete" delete-key="POLICY_MGR_URL" /> + <transfer operation="delete" delete-key="" /> + </task> + </pre-upgrade> + <upgrade> <task xsi:type="restart" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7e35039/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerConfigCalculationTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerConfigCalculationTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerConfigCalculationTest.java new file mode 100644 index 0000000..88fa58e --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerConfigCalculationTest.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.serveraction.upgrades; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; +import org.apache.ambari.server.actionmanager.HostRoleCommand; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.ConfigImpl; +import org.easymock.EasyMock; +import org.junit.Before; +import org.junit.Test; + +import com.google.inject.Injector; + +/** + * Tests upgrade-related server side actions + */ +public class RangerConfigCalculationTest { + + private Injector m_injector; + private Clusters m_clusters; + private Field m_clusterField; + + @Before + public void setup() throws Exception { + m_injector = EasyMock.createMock(Injector.class); + m_clusters = EasyMock.createMock(Clusters.class); + Cluster cluster = EasyMock.createMock(Cluster.class); + + Config adminConfig = new ConfigImpl("admin-properties") { + Map<String, String> mockProperties = new HashMap<String, String>() {{ + put("DB_FLAVOR", "MYSQL"); + put("db_host", "host1"); + put("db_name", "ranger"); + }}; + @Override + public Map<String, String> getProperties() { + return mockProperties; + } + }; + + Config adminSiteConfig = new ConfigImpl("admin-properties") { + Map<String, String> mockProperties = new HashMap<String, String>(); + @Override + public Map<String, String> getProperties() { + return mockProperties; + } + + @Override + public void setProperties(Map<String, String> properties) { + mockProperties.putAll(properties); + } + + @Override + public void persist(boolean newConfig) { + // no-op + } + }; + + expect(cluster.getDesiredConfigByType("admin-properties")).andReturn(adminConfig).atLeastOnce(); + expect(cluster.getDesiredConfigByType("ranger-admin-site")).andReturn(adminSiteConfig).atLeastOnce(); + + expect(m_clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes(); + expect(m_injector.getInstance(Clusters.class)).andReturn(m_clusters).atLeastOnce(); + + replay(m_injector, m_clusters, cluster); + + m_clusterField = RangerConfigCalculation.class.getDeclaredField("m_clusters"); + m_clusterField.setAccessible(true); + } + + @Test + public void testAction() throws Exception { + + Map<String, String> commandParams = new HashMap<String, String>(); + commandParams.put("clusterName", "c1"); + + ExecutionCommand executionCommand = new ExecutionCommand(); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName("c1"); + + HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class); + expect(hrc.getRequestId()).andReturn(1L).anyTimes(); + expect(hrc.getStageId()).andReturn(2L).anyTimes(); + expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes(); + replay(hrc); + + RangerConfigCalculation action = new RangerConfigCalculation(); + m_clusterField.set(action, m_clusters); + + action.setExecutionCommand(executionCommand); + action.setHostRoleCommand(hrc); + + CommandReport report = action.execute(null); + assertNotNull(report); + + Cluster c = m_clusters.getCluster("c1"); + Config config = c.getDesiredConfigByType("ranger-admin-site"); + Map<String, String> map = config.getProperties(); + + assertTrue(map.containsKey("ranger.jpa.jdbc.driver")); + assertTrue(map.containsKey("ranger.jpa.jdbc.url")); + assertTrue(map.containsKey("ranger.jpa.jdbc.dialect")); + assertTrue(map.containsKey("ranger.jpa.audit.jdbc.driver")); + assertTrue(map.containsKey("ranger.jpa.audit.jdbc.url")); + assertTrue(map.containsKey("ranger.jpa.audit.jdbc.dialect")); + + assertEquals("com.mysql.jdbc.Driver", map.get("ranger.jpa.jdbc.driver")); + assertEquals("jdbc:mysql://host1/ranger", map.get("ranger.jpa.jdbc.url")); + assertEquals("org.eclipse.persistence.platform.database.MySQLPlatform", map.get("ranger.jpa.jdbc.dialect")); + + assertEquals("com.mysql.jdbc.Driver", map.get("ranger.jpa.audit.jdbc.driver")); + assertEquals("jdbc:mysql://host1/ranger", map.get("ranger.jpa.audit.jdbc.url")); + assertEquals("org.eclipse.persistence.platform.database.MySQLPlatform", map.get("ranger.jpa.audit.jdbc.dialect")); + + config = c.getDesiredConfigByType("admin-properties"); + config.getProperties().put("DB_FLAVOR", "oracle"); + + report = action.execute(null); + assertNotNull(report); + + config = c.getDesiredConfigByType("ranger-admin-site"); + map = config.getProperties(); + + assertEquals("oracle.jdbc.OracleDriver", map.get("ranger.jpa.jdbc.driver")); + assertEquals("jdbc:oracle:thin:@//host1", map.get("ranger.jpa.jdbc.url")); + assertEquals("org.eclipse.persistence.platform.database.OraclePlatform", map.get("ranger.jpa.jdbc.dialect")); + + assertEquals("oracle.jdbc.OracleDriver", map.get("ranger.jpa.audit.jdbc.driver")); + assertEquals("jdbc:oracle:thin:@//host1", map.get("ranger.jpa.audit.jdbc.url")); + assertEquals("org.eclipse.persistence.platform.database.OraclePlatform", map.get("ranger.jpa.audit.jdbc.dialect")); + + } + + +}