Repository: ambari Updated Branches: refs/heads/branch-2.2 2d8d76329 -> 00134fe76 refs/heads/trunk 63ec0ea4b -> 67edc4079
AMBARI-14245. AMBARI-14245 : SYNC SOURCE property needs to be maintained for Manual Upgrade using upgradeHelper (2.2.x to 2.3.x) (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/67edc407 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/67edc407 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/67edc407 Branch: refs/heads/trunk Commit: 67edc4079e26304a914520bae1337702c3957ede Parents: 63ec0ea Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Mon Dec 7 22:04:51 2015 +0200 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Mon Dec 7 22:04:51 2015 +0200 ---------------------------------------------------------------------- ambari-server/src/main/python/upgradeHelper.py | 36 ++++++- .../0.4.0/configuration/usersync-properties.xml | 3 + .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 4 +- .../catalog/UpgradeCatalog_2.2_to_2.3.json | 107 ++++++++++++++++--- 4 files changed, 131 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/67edc407/ambari-server/src/main/python/upgradeHelper.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/upgradeHelper.py b/ambari-server/src/main/python/upgradeHelper.py index 1a6da53..31aa721 100644 --- a/ambari-server/src/main/python/upgradeHelper.py +++ b/ambari-server/src/main/python/upgradeHelper.py @@ -211,6 +211,8 @@ class Options(Const): ZK_OPTIONS = "zoo.cfg" KAFKA_BROKER_CONF = "kafka-broker" RANGER_ADMIN = "admin-properties" + RANGER_USERSYNC = "usersync-properties" + RANGER_ENV = "ranger-env" KAFKA_PORT = "port" RANGER_EXTERNAL_URL = "policymgr_external_url" ZK_CLIENTPORT = "clientPort" @@ -1514,6 +1516,28 @@ def get_hdfs_batch_filespool_dir(config_name, component): return path +def get_usersync_sync_source(): + ug_sync_source = 'org.apache.ranger.unixusersync.process.UnixUserGroupBuilder' + sync_source = 'unix' + if Options.server_config_factory is not None and Options.RANGER_USERSYNC in Options.server_config_factory.items(): + props = Options.server_config_factory.get_config(Options.RANGER_USERSYNC) + if "SYNC_SOURCE" in props.properties: + sync_source = props.properties['SYNC_SOURCE'] + + if sync_source == 'ldap': + ug_sync_source = 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder' + return ug_sync_source + +def get_audit_check(audit_type): + audit_check_flag = "false" + if Options.server_config_factory is not None and Options.RANGER_ENV in Options.server_config_factory.items(): + props = Options.server_config_factory.get_config(Options.RANGER_ENV) + audit_property = "xasecure.audit.destination.{0}".format(audit_type) + if audit_property in props.properties: + audit_check_flag = props.properties[audit_property] + + return audit_check_flag + def get_jt_host(catalog): """ :type catalog: UpgradeCatalog @@ -1554,11 +1578,13 @@ def get_ranger_service_details(): data['RANGER_JDBC_DIALECT'] = 'org.eclipse.persistence.platform.database.MySQLPlatform' data['RANGER_JDBC_URL'] = 'jdbc:mysql://{0}/{1}'.format(properties_latest['db_host'], properties_latest['db_name']) data['RANGER_AUDIT_JDBC_URL'] = 'jdbc:mysql://{0}/{1}'.format(properties_latest['db_host'], properties_latest['audit_db_name']) + data['RANGER_ROOT_JDBC_URL'] = 'jdbc:mysql://{0}'.format(properties_latest['db_host']) elif properties_latest['DB_FLAVOR'].lower() == 'oracle': data['RANGER_JDBC_DRIVER'] = 'oracle.jdbc.OracleDriver' data['RANGER_JDBC_DIALECT'] = 'org.eclipse.persistence.platform.database.OraclePlatform' data['RANGER_JDBC_URL'] = 'jdbc:oracle:thin:@//{0}'.format(properties_latest['db_host']) data['RANGER_AUDIT_JDBC_URL'] = 'jdbc:oracle:thin:@//{0}'.format(properties_latest['db_host']) + data['RANGER_ROOT_JDBC_URL'] = 'jdbc:oracle:thin:@//{0}'.format(properties_latest['db_host']) return data @@ -1762,8 +1788,6 @@ def _substitute_handler(upgrade_catalog, tokens, value): value = value.replace(token, get_audit_jdbc_url()) elif token == "{STORM_AUDIT_JDBC_URL}": value = value.replace(token, get_audit_jdbc_url()) - elif token == "{AUDIT_DB_PASSWD}": - value = value.replace(token, get_audit_db_passwd()) elif token == "{AUDIT_TO_DB_HDFS}": value = value.replace(token, get_audit_to_db_enabled("ranger-hdfs-plugin-properties")) elif token == "{AUDIT_TO_DB_HBASE}": @@ -1794,6 +1818,14 @@ def _substitute_handler(upgrade_catalog, tokens, value): value = value.replace(token, get_hdfs_batch_filespool_dir("ranger-knox-plugin-properties", "knox")) elif token == "{AUDIT_HDFS_FILESPOOL_DIR_STORM}": value = value.replace(token, get_hdfs_batch_filespool_dir("ranger-storm-plugin-properties", "storm")) + elif token == "{USERSYNC_SYNC_SOURCE}": + value = value.replace(token, get_usersync_sync_source()) + elif token == "{AUDIT_TO_DB}": + value = value.replace(token, get_audit_check("db")) + elif token == "{AUDIT_TO_HDFS}": + value = value.replace(token, get_audit_check("hdfs")) + elif token == "{RANGER_ROOT_JDBC_URL}": + value = value.replace(token, get_ranger_service_details()['RANGER_ROOT_JDBC_URL']) return value http://git-wip-us.apache.org/repos/asf/ambari/blob/67edc407/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/usersync-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/usersync-properties.xml b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/usersync-properties.xml index c551fc8..9ad6bbd 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/usersync-properties.xml +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/usersync-properties.xml @@ -107,6 +107,9 @@ <display-name>âUser Search Filter</display-name> <value></value> <description>default value is empty</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> </property> <property> <name>SYNC_LDAP_USER_NAME_ATTRIBUTE</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/67edc407/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index e958a3f..6f4f7c3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -737,7 +737,7 @@ <transfer operation="copy" from-type="ranger-knox-plugin-properties" from-key="XAAUDIT.HDFS.IS_ENABLED" to-key="xasecure.audit.destination.hdfs" default-value="true"/> <transfer operation="copy" from-type="ranger-knox-plugin-properties" from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" default-value="/var/log/knox/audit/hdfs/spool"/> <transfer operation="copy" from-type="ranger-knox-plugin-properties" from-key="XAAUDIT.DB.USER_NAME" to-key="xasecure.audit.destination.db.user" default-value=""/> - <transfer operation="copy" from-type="ranger-knox-plugin-properties" from-key="XAAUDIT.DB.PASSWORD" to-key="xasecure.audit.destination.db.passwordr" default-value="" mask="true"/> + <transfer operation="copy" from-type="ranger-knox-plugin-properties" from-key="XAAUDIT.DB.PASSWORD" to-key="xasecure.audit.destination.db.password" default-value="" mask="true"/> <set key="xasecure.audit.credential.provider.file" value="jceks://file{{credential_file}}"/> <set key="xasecure.audit.destination.solr" value="false"/> <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}"/> @@ -981,7 +981,7 @@ http://www.apache.org/licenses/LICENSE-2.0 <transfer operation="copy" from-type="ranger-storm-plugin-properties" from-key="XAAUDIT.HDFS.IS_ENABLED" to-key="xasecure.audit.destination.hdfs" default-value="true" /> <transfer operation="copy" from-type="ranger-storm-plugin-properties" from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" default-value="/var/log/storm/audit/hdfs/spool" /> <transfer operation="copy" from-type="ranger-storm-plugin-properties" from-key="XAAUDIT.DB.USER_NAME" to-key="xasecure.audit.destination.db.user" default-value=""/> - <transfer operation="copy" from-type="ranger-storm-plugin-properties" from-key="XAAUDIT.DB.PASSWORD" to-key="xasecure.audit.destination.db.passwordr" default-value="" mask="true"/> + <transfer operation="copy" from-type="ranger-storm-plugin-properties" from-key="XAAUDIT.DB.PASSWORD" to-key="xasecure.audit.destination.db.password" default-value="" mask="true"/> <set key="xasecure.audit.credential.provider.file" value="jceks://file{{credential_file}}"/> <set key="xasecure.audit.destination.solr" value="false"/> <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/67edc407/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json index 64849f7..de8f8e3 100644 --- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json +++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json @@ -601,11 +601,7 @@ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/hadoop/hdfs/audit/db/spool", "xasecure.audit.is.enabled": "true", "xasecure.audit.destination.db.password": { - "value": "{AUDIT_DB_PASSWD}", - "template": "yes", - "required-services": [ - "RANGER" - ] + "value": "{{xa_audit_db_password}}" }, "xasecure.audit.destination.db": { "value": "{AUDIT_TO_DB_HDFS}", @@ -728,8 +724,7 @@ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/hbase/audit/db/spool", "xasecure.audit.is.enabled": "true", "xasecure.audit.destination.db.password": { - "value": "{AUDIT_DB_PASSWD}", - "template": "yes" + "value": "{{xa_audit_db_password}}" }, "xasecure.audit.destination.db": { "value": "{AUDIT_TO_DB_HBASE}", @@ -789,7 +784,7 @@ "ranger-hive-security": { "upgrade.transition": "yes", "ranger.plugin.hive.policy.source.impl": "org.apache.ranger.admin.client.RangerAdminRESTClient", - "ranger.plugin.hive.policy.rest.ssl.config.file": "usr/hdp/current/hive-server2/conf/conf.server/ranger-policymgr-ssl.xml", + "ranger.plugin.hive.policy.rest.ssl.config.file": "/usr/hdp/current/hive-server2/conf/conf.server/ranger-policymgr-ssl.xml", "ranger.plugin.hive.policy.pollIntervalMs": "30000", "ranger.plugin.hive.policy.cache.dir": { "value": "{RANGER_PLUGIN_HIVE_POLICY_CACHE_DIR}", @@ -827,8 +822,7 @@ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/hive/audit/db/spool", "xasecure.audit.is.enabled": "true", "xasecure.audit.destination.db.password": { - "value": "{AUDIT_DB_PASSWD}", - "template": "yes" + "value": "{{xa_audit_db_password}}" }, "xasecure.audit.destination.db": { "value": "{AUDIT_TO_DB_HIVE}", @@ -925,8 +919,7 @@ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/knox/audit/db/spool", "xasecure.audit.is.enabled": "true", "xasecure.audit.destination.db.password": { - "value": "{AUDIT_DB_PASSWD}", - "template": "yes" + "value": "{{xa_audit_db_password}}" }, "xasecure.audit.destination.db": { "value": "{AUDIT_TO_DB_KNOX}", @@ -1023,8 +1016,7 @@ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/storm/audit/db/spool", "xasecure.audit.is.enabled": "true", "xasecure.audit.destination.db.password": { - "value": "{AUDIT_DB_PASSWD}", - "template": "yes" + "value": "{{xa_audit_db_password}}" }, "xasecure.audit.destination.db": { "value": "{AUDIT_TO_DB_STORM}", @@ -1178,6 +1170,30 @@ }, "xml_configurations_supported": { "value": "true" + }, + "ranger_privelege_user_jdbc_url": { + "value": "{RANGER_ROOT_JDBC_URL}", + "template": "yes" + }, + "ranger-yarn-plugin-enabled": { + "value": "No" + }, + "ranger-kafka-plugin-enabled":{ + "value": "No" + }, + "xasecure.audit.destination.solr":{ + "value": "false" + }, + "is_solrCloud_enabled":{ + "value": "false" + }, + "xasecure.audit.destination.hdfs": { + "value": "{AUDIT_TO_HDFS}", + "template": "yes" + }, + "xasecure.audit.destination.db": { + "value": "{AUDIT_TO_DB}", + "template": "yes" } }, "ranger-admin-site": { @@ -1272,6 +1288,57 @@ }, "ranger.jpa.audit.jdbc.password": { "value": "_" + }, + "ranger.sso.providerurl": { + "value": "" + }, + "ranger.sso.publicKey": { + "value": "" + }, + "ranger.sso.cookiename": { + "value": "hadoop-jwt" + }, + "ranger.sso.enabled": { + "value": "false" + }, + "ranger.sso.query.param.originalurl": { + "value": "originalUrl" + }, + "ranger.sso.browser.useragent": { + "value": "Mozilla,chrome" + }, + "ranger.ldap.user.searchfilter":{ + "value": "{{ranger_ug_ldap_user_searchfilter}}" + }, + "ranger.ldap.group.roleattribute":{ + "value": "cn" + }, + "ranger.ldap.base.dn":{ + "value": "dc=example,dc=com" + }, + "ranger.ldap.bind.dn":{ + "value": "{{ranger_ug_ldap_bind_dn}}" + }, + "ranger.ldap.bind.password":{ + "value": "{{ranger_usersync_ldap_ldapbindpassword}}" + }, + "ranger.ldap.referral":{ + "value": "ignore" + }, + "ranger.ldap.ad.base.dn":{ + "value": "dc=example,dc=com" + }, + "ranger.ldap.ad.bind.dn":{ + "value": "{{ranger_ug_ldap_bind_dn}}" + }, + "ranger.ldap.ad.bind.password":{ + "value": "{{ranger_usersync_ldap_ldapbindpassword}}" + }, + "ranger.ldap.ad.user.searchfilter":{ + "value": "{{ranger_ug_ldap_user_searchfilter}}" + }, + "ranger.ldap.ad.referral":{ + "value": "ignore" } }, "ranger-ugsync-site": { @@ -1355,13 +1422,23 @@ "value": "500" }, "ranger.usersync.source.impl.class": { - "value": "" + "value": "{USERSYNC_SYNC_SOURCE}", + "template": "yes" }, "ranger.usersync.policymanager.baseURL": { "value": "{{ranger_external_url}}" }, "ranger.usersync.sleeptimeinmillisbetweensynccycle": { "value": "60000" + }, + "ranger.usersync.ldap.referral": { + "value": "ignore" + }, + "ranger.usersync.unix.group.file": { + "value": "/etc/group" + }, + "ranger.usersync.unix.password.file": { + "value": "/etc/passwd" } }, "usersync-properties":{