Repository: ambari Updated Branches: refs/heads/trunk 1fce0cabb -> b571e4a0b
AMBARI-16204 - Atlas Integration : Atlas start failed after removing old version of the atlas and installing new one Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b571e4a0 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b571e4a0 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b571e4a0 Branch: refs/heads/trunk Commit: b571e4a0b05d813adfcbbfe7f90563555844ca28 Parents: 1fce0ca Author: tbeerbower <tbeerbo...@hortonworks.com> Authored: Tue May 3 06:03:56 2016 -0400 Committer: tbeerbower <tbeerbo...@hortonworks.com> Committed: Tue May 3 06:04:28 2016 -0400 ---------------------------------------------------------------------- .../configuration/application-properties.xml | 6 ++--- .../ATLAS/0.1.0.2.3/metainfo.xml | 6 ++--- .../package/files/users-credentials.properties | 21 +++++++++++++++++ .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 24 +++++++++++++------- .../package/scripts/metadata_server.py | 9 ++++++++ .../ATLAS/0.1.0.2.3/package/scripts/params.py | 2 ++ .../configuration/application-properties.xml | 10 ++++++++ .../services/ATLAS/configuration/atlas-env.xml | 10 ++++++++ .../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 4 ++-- .../stacks/2.3/ATLAS/test_metadata_server.py | 6 +++++ .../test/python/stacks/2.3/configs/default.json | 3 ++- .../test/python/stacks/2.3/configs/secure.json | 3 ++- .../stacks/2.5/ATLAS/test_atlas_server.py | 6 +++++ .../test/python/stacks/2.5/configs/default.json | 3 ++- 14 files changed, 94 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml index f3ea36f..bf0467e 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml @@ -21,12 +21,12 @@ <configuration supports_final="false"> <property> <name>atlas.graph.storage.backend</name> - <value>hbase</value> + <value>berkeleyje</value> <description></description> </property> <property> - <name>atlas.graph.storage.hostname</name> - <value>localhost:{{atlas_hbase_zk_port}}</value> + <name>atlas.graph.storage.directory</name> + <value>/var/lib/atlas/data/berkeley</value> <description></description> </property> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml index 5d7766b..f4115f7 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml @@ -46,7 +46,7 @@ </logs> <configFile> <type>properties</type> - <fileName>application.properties.xml</fileName> + <fileName>application.properties</fileName> <dictionaryName>application-properties</dictionaryName> </configFile> <configFile> @@ -61,7 +61,7 @@ </configFile> <configFile> <type>xml</type> - <fileName>atlas-hbase-site.xml</fileName> + <fileName>hbase-site.xml</fileName> <dictionaryName>atlas-hbase-site</dictionaryName> </configFile> </component> @@ -80,7 +80,7 @@ </commandScript> <configFile> <type>properties</type> - <fileName>application.properties.xml</fileName> + <fileName>application.properties</fileName> <dictionaryName>application-properties</dictionaryName> </configFile> <configFile> http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties new file mode 100644 index 0000000..4d264a0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#username=password +admin=admin +user=user123 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index 43c1ea6..8e2fcc2 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -107,14 +107,22 @@ def metadata(): content=InlineTemplate(params.metadata_log4j_content) ) - # hbase-site for embedded hbase used by Atlas - XmlConfig( "hbase-site.xml", - conf_dir = params.atlas_hbase_conf_dir, - configurations = params.config['configurations']['atlas-hbase-site'], - configuration_attributes=params.config['configuration_attributes']['atlas-hbase-site'], - owner = params.metadata_user, - group = params.user_group - ) + File(format("{conf_dir}/users-credentials.properties"), + mode=0644, + owner=params.metadata_user, + group=params.user_group, + content=StaticFile('users-credentials.properties') + ) + + if params.atlas_has_embedded_hbase: + # hbase-site for embedded hbase used by Atlas + XmlConfig( "hbase-site.xml", + conf_dir = params.atlas_hbase_conf_dir, + configurations = params.config['configurations']['atlas-hbase-site'], + configuration_attributes=params.config['configuration_attributes']['atlas-hbase-site'], + owner = params.metadata_user, + group = params.user_group + ) if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py index 4b08cbc..1e9e7a7 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py @@ -27,6 +27,8 @@ from resource_management.libraries.functions.security_commons import build_expec from resource_management.libraries.functions.show_logs import show_logs from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import StackFeature +import os +import shutil class MetadataServer(Script): @@ -34,6 +36,13 @@ class MetadataServer(Script): return "atlas-server" def install(self, env): + import params + + expanded_atlas = os.path.join(params.expanded_war_dir, "atlas") + + if os.path.exists(expanded_atlas): + shutil.rmtree(expanded_atlas) + self.install_packages(env) def configure(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index defd6f0..51fa039 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -68,6 +68,8 @@ atlas_hbase_log_dir = os.path.join(metadata_home, "hbase", "logs") atlas_hbase_data_dir = os.path.join(metadata_home, "data") atlas_hbase_zk_port = default("/configurations/atlas-hbase-site/hbase.zookeeper.property.clientPort", None) +atlas_has_embedded_hbase = default("/configurations/atlas-env/has_embedded_hbase", False) + # service locations hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOME' in os.environ else '/etc/hadoop/conf' http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml index 8412fc0..78600ee 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml @@ -43,6 +43,16 @@ </value-attributes> </property> <property> + <name>atlas.graph.storage.backend</name> + <value>hbase</value> + <description></description> + </property> + <property> + <name>atlas.graph.storage.hostname</name> + <value>localhost:{{atlas_hbase_zk_port}}</value> + <description></description> + </property> + <property> <name>atlas.graph.index.search.backend</name> <value>solr5</value> <description>The Atlas indexing backend (e.g. solr5).</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml index a2ea7e5..bae4de3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml @@ -70,6 +70,16 @@ <visible>false</visible> </value-attributes> </property> + <property> + <name>has_embedded_hbase</name> + <value>true</value> + <description>Indicates whether or not Atlas can install embedded Hbase.</description> + <value-attributes> + <read-only>true</read-only> + <overridable>false</overridable> + <visible>false</visible> + </value-attributes> + </property> <!-- metadata-env.sh --> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml index 92f198a..c787b40 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml @@ -27,7 +27,7 @@ <cardinality>1+</cardinality> <configFile> <type>properties</type> - <fileName>application.properties.xml</fileName> + <fileName>atlas-application.properties</fileName> <dictionaryName>application-properties</dictionaryName> </configFile> <logs> @@ -42,7 +42,7 @@ <name>ATLAS_CLIENT</name> <configFile> <type>properties</type> - <fileName>application.properties.xml</fileName> + <fileName>atlas-application.properties</fileName> <dictionaryName>application-properties</dictionaryName> </configFile> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py index cf18f92..8ae2c11 100644 --- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py +++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py @@ -110,6 +110,12 @@ class TestMetadataServer(RMFTestCase): group='hadoop', mode=0644, ) + self.assertResourceCalled('File', '/etc/atlas/conf/users-credentials.properties', + content=StaticFile('users-credentials.properties'), + owner='atlas', + group='hadoop', + mode=0644, + ) self.assertResourceCalled('XmlConfig', 'hbase-site.xml', owner = 'atlas', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/test/python/stacks/2.3/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json index d423ce5..c8b418b 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json @@ -210,7 +210,8 @@ "metadata_log_dir": "/var/log/atlas", "metadata_data_dir": "/var/lib/atlas/data", "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp", - "metadata_conf_file": "application.properties" + "metadata_conf_file": "application.properties", + "has_embedded_hbase":"true" }, "atlas-log4j": { "content": "<property><name>content</name><description>Custom log4j.properties</description><value></value></property>", http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/test/python/stacks/2.3/configs/secure.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/secure.json b/ambari-server/src/test/python/stacks/2.3/configs/secure.json index 89ab6f0..7ebcedf 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/secure.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/secure.json @@ -202,7 +202,8 @@ "metadata_log_dir": "/var/log/atlas", "metadata_data_dir": "/var/lib/atlas/data", "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp", - "metadata_conf_file": "application.properties" + "metadata_conf_file": "application.properties", + "has_embedded_hbase":"true" }, "atlas-log4j": { "content": "<property><name>content</name><description>Custom log4j.properties</description><value></value></property>", http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py index 83d4c5b..422b1c9 100644 --- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py +++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py @@ -108,6 +108,12 @@ class TestAtlasServer(RMFTestCase): group='hadoop', mode=0644, ) + self.assertResourceCalled('File', '/etc/atlas/conf/users-credentials.properties', + content=StaticFile('users-credentials.properties'), + owner='atlas', + group='hadoop', + mode=0644, + ) self.assertResourceCalled('XmlConfig', 'hbase-site.xml', owner = 'atlas', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/b571e4a0/ambari-server/src/test/python/stacks/2.5/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/default.json b/ambari-server/src/test/python/stacks/2.5/configs/default.json index 5188730..e4a97f6 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/default.json @@ -210,7 +210,8 @@ "metadata_log_dir": "/var/log/atlas", "metadata_data_dir": "/var/lib/atlas/data", "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp", - "metadata_conf_file": "atlas-application.properties" + "metadata_conf_file": "atlas-application.properties", + "has_embedded_hbase":"true" }, "atlas-log4j": { "content": "<property><name>content</name><description>Custom log4j.properties</description><value></value></property>",
