Repository: ambari Updated Branches: refs/heads/trunk e973a534f -> fdaa98043
AMBARI-11732. The properties listed in Atlas configuration panels of Ambari UI differed in name than actual property names in configuration files (Jon Marin via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fdaa9804 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fdaa9804 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fdaa9804 Branch: refs/heads/trunk Commit: fdaa98043ffbd3b6322c8a68e9d8417cf5b450af Parents: e973a53 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Fri Jun 5 19:03:11 2015 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Fri Jun 5 19:03:11 2015 -0700 ---------------------------------------------------------------------- .../configuration/application-properties.xml | 109 +++++-------------- .../ATLAS/0.1.0.2.3/kerberos.json | 12 +- .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 6 +- .../ATLAS/0.1.0.2.3/package/scripts/params.py | 11 +- .../stacks/2.3/ATLAS/test_metadata_server.py | 11 +- .../test/python/stacks/2.3/configs/default.json | 43 ++++---- 6 files changed, 66 insertions(+), 126 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fdaa9804/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml index 29f4e72..db5ffd1 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml @@ -20,172 +20,115 @@ --> <configuration supports_final="false"> <property> - <name>graph_storage_backend</name> + <name>metadata.graph.storage.backend</name> <value>berkeleyje</value> <description></description> </property> <property> - <name>graph_storage_directory</name> + <name>metadata.graph.storage.directory</name> <value>/var/lib/atlas/data/berkeley</value> <description></description> </property> <property> - <name>graph_index_search_backend</name> + <name>metadata.graph.index.search.backend</name> <value>elasticsearch</value> <description></description> </property> <property> - <name>graph_index_search_directory</name> + <name>metadata.graph.index.search.directory</name> <value>/var/lib/atlas/data/es</value> <description></description> </property> <property> - <name>graph_index_search_elasticsearch_client_only</name> + <name>metadata.graph.index.search.elasticsearch.client-only</name> <value>false</value> <description></description> </property> <property> - <name>graph_index_search_elasticsearch_local_mode</name> + <name>metadata.graph.index.search.elasticsearch.local-mode</name> <value>true</value> <description></description> </property> <property> - <name>lineage_hive_table_type_name</name> + <name>metadata.lineage.hive.table.type.name</name> <value>Table</value> <description></description> </property> <property> - <name>lineage_hive_column_type_name</name> + <name>metadata.lineage.hive.column.type.name</name> <value>Column</value> <description></description> </property> <property> - <name>lineage_hive_table_column_name</name> + <name>metadata.lineage.hive.table.column.name</name> <value>columns</value> <description></description> </property> <property> - <name>lineage_hive_process_type_name</name> + <name>metadata.lineage.hive.process.type.name</name> <value>LoadProcess</value> <description></description> </property> <property> - <name>lineage_hive_process_inputs_name</name> + <name>metadata.lineage.hive.process.inputs.name</name> <value>inputTables</value> <description></description> </property> <property> - <name>lineage_hive_process_outputs_name</name> + <name>metadata.lineage.hive.process.outputs.name</name> <value>outputTables</value> <description></description> </property> <property> - <name>enableTLS</name> + <name>metadata.enableTLS</name> <value>false</value> <description></description> </property> <property> - <name>authentication_method</name> + <name>metadata.authentication.method</name> <value>simple</value> <description></description> </property> <property> - <name>authentication_principal</name> + <name>metadata.authentication.principal</name> <value>atlas</value> <description></description> </property> <property> - <name>authentication_keytab</name> + <name>metadata.authentication.keytab</name> <value>/etc/security/keytabs/atlas.service.keytab</value> <description></description> </property> <property> - <name>http_authentication_enabled</name> + <name>metadata.server.bind.address</name> + <value>localhost</value> + <description></description> + </property> + <property> + <name>metadata.http.authentication.enabled</name> <value>false</value> <description></description> </property> <property> - <name>http_authentication_type</name> + <name>metadata.http.authentication.type</name> <value>simple</value> <description></description> </property> <property> - <name>http_authentication_kerberos_principal</name> + <name>metadata.http.authentication.kerberos.principal</name> <value>HTTP/_h...@example.com</value> <description></description> </property> <property> - <name>http_authentication_kerberos_keytab</name> + <name>metadata.http.authentication.kerberos.keytab</name> <value>/etc/security/keytabs/spnego.service.keytab</value> <description></description> </property> <property> - <name>http_authentication_kerberos_name_rules</name> + <name>metadata.http.authentication.kerberos.name.rules</name> <value>RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*// DEFAULT</value> <description></description> </property> - <!-- application.properties --> - <property> - <name>content</name> - <description>This is the jinja template for application.properties file</description> - <value> - # - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - # - - ######### Graph Database Configs ######### - # Graph Storage - metadata.graph.storage.backend={{graph_storage_backend}} - metadata.graph.storage.directory={{graph_storage_directory}} - - # Graph Search Index - metadata.graph.index.search.backend={{graph_index_search_backend}} - metadata.graph.index.search.directory={{graph_index_search_directory}} - metadata.graph.index.search.elasticsearch.client-only={{graph_index_search_elasticsearch_client_only}} - metadata.graph.index.search.elasticsearch.local-mode={{graph_index_search_elasticsearch_local_mode}} - - ######### Hive Lineage Configs ######### - metadata.lineage.hive.table.type.name={{lineage_hive_table_type_name}} - metadata.lineage.hive.column.type.name={{lineage_hive_column_type_name}} - metadata.lineage.hive.table.column.name={{lineage_hive_table_column_name}} - metadata.lineage.hive.process.type.name={{lineage_hive_process_type_name}} - metadata.lineage.hive.process.inputs.name={{lineage_hive_process_inputs_name}} - metadata.lineage.hive.process.outputs.name={{lineage_hive_process_outputs_name}} - ######### Security Properties ######### - - # SSL config - metadata.enableTLS={{enableTLS}} - - # Service Authentication - metadata.authentication.method={{authentication_method}} - metadata.authentication.principal={{authentication_principal}} - metadata.authentication.keytab={{authentication_keytab}} - metadata.server.bind.address={{metadata_host}} - - # SPNEGO - metadata.http.authentication.enabled={{http_authentication_enabled}} - metadata.http.authentication.type={{http_authentication_type}} - metadata.http.authentication.kerberos.principal={{http_authentication_kerberos_principal}} - metadata.http.authentication.kerberos.keytab={{http_authentication_kerberos_keytab}} - metadata.http.authentication.kerberos.name.rules={{http_authentication_kerberos_name_rules}} - - ######### Security Properties ######### - </value> - </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/fdaa9804/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json index b57f565..778b23c 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json @@ -5,14 +5,14 @@ "configurations": [ { "application-properties": { - "authentication_method": "kerberos", - "http_authentication_enabled": "true", - "http_authentication_type": "kerberos" + "metadata.authentication.method": "kerberos", + "metadata.http.authentication.enabled": "true", + "metadata.http.authentication.type": "kerberos" } } ], "auth_to_local_properties" : [ - "application-properties/http_authentication_kerberos_name_rules" + "application-properties/metadata.http.authentication.kerberos.name.rules" ], "components": [ { @@ -23,7 +23,7 @@ "principal": { "value": "atlas/_HOST@${realm}", "type" : "service", - "configuration": "application-properties/authentication_principal", + "configuration": "application-properties/metadata.authentication.principal", "local_username" : "${metadata-env/metadata_user}" }, "keytab": { @@ -36,7 +36,7 @@ "name": "${cluster-env/user_group}", "access": "" }, - "configuration": "application-properties/authentication_keytab" + "configuration": "application-properties/metadata.authentication.keytab" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/fdaa9804/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index a4274ca..dc9daa9 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -20,7 +20,7 @@ limitations under the License. import os import shutil from resource_management import Directory, Fail, Logger, File, \ - InlineTemplate, StaticFile + InlineTemplate, PropertiesFile, StaticFile from resource_management.libraries.functions import format @@ -74,8 +74,8 @@ def metadata(): Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir)) shutil.copy2(metadata_war_file, params.expanded_war_dir) - File(format('{conf_dir}/application.properties'), - content=InlineTemplate(params.application_properties_content), + PropertiesFile(format('{conf_dir}/application.properties'), + properties = params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group http://git-wip-us.apache.org/repos/asf/ambari/blob/fdaa9804/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index 577c5ac..c77083b 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -66,16 +66,11 @@ metadata_port = config['configurations']['metadata-env']['metadata_port'] metadata_host = config['hostname'] # application properties -application_properties = config['configurations']['application-properties'] - -for key, value in application_properties.iteritems(): - # fix the multi-line property - if (key == 'http_authentication_kerberos_name_rules'): - value = ' \\ \n'.join(value.splitlines()) - globals()[key] = value +application_properties = dict(config['configurations']['application-properties']) +application_properties['metadata.http.authentication.kerberos.name.rules'] = ' \\ \n'.join(application_properties['metadata.http.authentication.kerberos.name.rules'].splitlines()) +application_properties['metadata.server.bind.address'] = metadata_host metadata_env_content = config['configurations']['metadata-env']['content'] -application_properties_content = config['configurations']['application-properties']['content'] metadata_opts = config['configurations']['metadata-env']['metadata_opts'] metadata_classpath = config['configurations']['metadata-env']['metadata_classpath'] http://git-wip-us.apache.org/repos/asf/ambari/blob/fdaa9804/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py index 9db21f8..81ee91b 100644 --- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py +++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py @@ -64,11 +64,14 @@ class TestMetadataServer(RMFTestCase): cd_access='a', mode=0644 ) - self.assertResourceCalled('File', + appprops = dict(self.getConfig()['configurations'][ + 'application-properties']) + appprops['metadata.http.authentication.kerberos.name.rules'] = ' \\ \n'.join(appprops['metadata.http.authentication.kerberos.name.rules'].splitlines()) + appprops['metadata.server.bind.address'] = 'c6401.ambari.apache.org' + + self.assertResourceCalled('PropertiesFile', '/etc/atlas/conf/application.properties', - content=InlineTemplate( - self.getConfig()['configurations'][ - 'application-properties']['content']), + properties=appprops, owner='atlas', group='hadoop', mode=0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/fdaa9804/ambari-server/src/test/python/stacks/2.3/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json index 424afdc..1df1643 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json @@ -161,28 +161,27 @@ "log.retention.hours": "168" }, "application-properties": { - "graph_storage_backend": "berkeleyje", - "graph_storage_directory": "data/berkley", - "graph_index_search_backend": "elasticsearch", - "graph_index_search_directory": "data/es", - "graph_index_search_elasticsearch_client_only": false, - "graph_index_search_elasticsearch_local_mode": true, - "lineage_hive_table_type_name": "Table", - "lineage_hive_column_type_name": "Column", - "lineage_hive_table_column_name": "columns", - "lineage_hive_process_type_name": "LoadProcess", - "lineage_hive_process_inputs_name": "inputTables", - "lineage_hive_process_outputs_name": "outputTables", - "enableTLS": false, - "authentication_method": "simple", - "authentication_principal": "atlas", - "authentication_keytab": "/etc/security/keytabs/atlas.service.keytab", - "http_authentication_enabled": false, - "http_authentication_type": "simple", - "http_authentication_kerberos_principal": "HTTP/_h...@example.com", - "http_authentication_kerberos_keytab": "/etc/security/keytabs/spnego.service.keytab", - "http_authentication_kerberos_name_rules": "DEFAULT", - "content": " # Graph Storage\nmetadata.graph.storage.backend={{graph_storage_backend}}\nmetadata.graph.storage.directory={{graph_storage_directory}}\nmetadata.graph.index.search.backend={{graph_index_search_backend}}\nmetadata.graph.index.search.directory={{graph_index_search_directory}}\nmetadata.graph.index.search.elasticsearch.client-only={{graph_index_search_elasticsearch_client_only}}\nmetadata.graph.index.search.elasticsearch.local-mode={{graph_index_search_elasticsearch_local_mode}}\nmetadata.lineage.hive.table.type.name={{lineage_hive_table_type_name}}\nmetadata.lineage.hive.column.type.name={{lineage_hive_column_type_name}}\nmetadata.lineage.hive.table.column.name={{lineage_hive_table_column_name}}\nmetadata.lineage.hive.process.type.name={{lineage_hive_process_type_name}}\nmetadata.lineage.hive.process.inputs.name={{lineage_hive_process_inputs_name}}\nmetadata.lineage.hive.process.outputs.name={{lineage_hive_process_outputs_name}}\nmetadata.enableTLS={{enableTLS}} \nmetadata.authentication.method={{authentication_method}}\nmetadata.authentication.principal={{authentication_principal}}\nmetadata.authentication.keytab={{authentication_keytab}}\nmetadata.http.authentication.enabled={{http_authentication_enabled}}\nmetadata.http.authentication.type={{http_authentication_type}}\nmetadata.http.authentication.kerberos.principal={{http_authentication_kerberos_principal}}\nmetadata.http.authentication.kerberos.keytab={{http_authentication_kerberos_keytab}}\nmetadata.http.authentication.kerberos.name.rules={{http_authentication_kerberos_name_rules}}" + "metadata.graph.storage.backend": "berkeleyje", + "metadata.graph.storage.directory": "data/berkley", + "metadata.graph.index.search.backend": "elasticsearch", + "metadata.graph.index.search.directory": "data/es", + "metadata.graph.index.search.elasticsearch.client-only": false, + "metadata.graph.index.search.elasticsearch.local-mode": true, + "metadata.lineage.hive.table.type.name": "Table", + "metadata.lineage.hive.column.type.name": "Column", + "metadata.lineage.hive.table.column.name": "columns", + "metadata.lineage.hive.process.type.name": "LoadProcess", + "metadata.lineage.hive.process.inputs.name": "inputTables", + "metadata.lineage.hive.process.outputs.name": "outputTables", + "metadata.enableTLS": false, + "metadata.authentication.method": "simple", + "metadata.authentication.principal": "atlas", + "metadata.authentication.keytab": "/etc/security/keytabs/atlas.service.keytab", + "metadata.http.authentication.enabled": false, + "metadata.http.authentication.type": "simple", + "metadata.http.authentication.kerberos.principal": "HTTP/_h...@example.com", + "metadata.http.authentication.kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab", + "metadata.http.authentication.kerberos.name.rules": "DEFAULT" }, "metadata-env": { "content": "# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path\nexport JAVA_HOME={{java64_home}}\n# any additional java opts you want to set. This will apply to both client and server operations\nexport METADATA_OPTS={{metadata_opts}}\n# metadata configuration directory\nexport METADATA_CONF={{conf_dir}}\n# Where log files are stored. Defatult is logs directory under the base install location\nexport METADATA_LOG_DIR={{log_dir}}\n# additional classpath entries\nexport METADATACPPATH={{metadata_classpath}}\n# data dir\nexport METADATA_DATA_DIR={{data_dir}}\n# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.\nexport METADATA_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}",