Repository: ambari Updated Branches: refs/heads/trunk 754291661 -> 14e4956f1
AMBARI-16853 - Atlas Integration : Atlas support for Logsearch Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/14e4956f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/14e4956f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/14e4956f Branch: refs/heads/trunk Commit: 14e4956f14d10b0ac37400b5d954ce08296df835 Parents: 7542916 Author: tbeerbower <tbeerbo...@hortonworks.com> Authored: Thu May 26 10:24:59 2016 -0400 Committer: tbeerbower <tbeerbo...@hortonworks.com> Committed: Thu May 26 10:24:59 2016 -0400 ---------------------------------------------------------------------- .../configuration/application-properties.xml | 30 --------- .../ATLAS/0.1.0.2.3/configuration/atlas-env.xml | 2 +- .../configuration/atlas-hbase-site.xml | 58 ---------------- .../ATLAS/0.1.0.2.3/metainfo.xml | 15 ----- .../0.1.0.2.3/package/scripts/atlas_client.py | 2 +- .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 70 +++++++++++--------- .../ATLAS/0.1.0.2.3/package/scripts/params.py | 26 +++++--- .../configuration/application-properties.xml | 33 ++++++++- .../services/ATLAS/configuration/atlas-env.xml | 19 +++--- .../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 16 ++++- .../stacks/HDP/2.5/services/stack_advisor.py | 58 +++++++++++++++- .../stacks/2.3/ATLAS/test_metadata_server.py | 28 ++------ .../test/python/stacks/2.3/configs/default.json | 6 +- .../test/python/stacks/2.3/configs/secure.json | 6 +- .../stacks/2.5/ATLAS/test_atlas_server.py | 29 +++----- .../stacks/2.5/common/test_stack_advisor.py | 16 ++++- .../test/python/stacks/2.5/configs/default.json | 6 +- 17 files changed, 212 insertions(+), 208 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml index bf0467e..ca8e4c5 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml @@ -136,31 +136,6 @@ <description>Indicates whether or not the notification service should be embedded.</description> </property> <property> - <name>atlas.kafka.data</name> - <value>{{metadata_home}}/data/kafka</value> - <description>The Kafka data directory.</description> - </property> - <property> - <name>atlas.kafka.bootstrap.servers</name> - <value>{{kafka_bootstrap_servers}}</value> - <description>Comma separated list of Kafka broker endpoints in host:port form</description> - </property> - <property> - <name>atlas.kafka.zookeeper.connect</name> - <value>{{kafka_zookeeper_connect}}</value> - <description>Comma separated list of servers forming Zookeeper quorum used by Kafka.</description> - </property> - <property> - <name>atlas.kafka.hook.group.id</name> - <value>atlas</value> - <description>Kafka group id for the hook topic.</description> - </property> - <property> - <name>atlas.kafka.entities.group.id</name> - <value>entities</value> - <description>Kafka group id for the entity topic.</description> - </property> - <property> <name>atlas.cluster.name</name> <value>{{cluster_name}}</value> <description>The cluster name.</description> @@ -220,9 +195,4 @@ <value>1000</value> <description></description> </property> - <property> - <name>atlas.audit.hbase.zookeeper.quorum</name> - <value>localhost:{{atlas_hbase_zk_port}}</value> - <description></description> - </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml index dd4b3e2..d52d1a3 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml @@ -103,7 +103,7 @@ export METADATA_DATA_DIR={{data_dir}} export METADATA_PID_DIR={{pid_dir}} # hbase conf dir -export HBASE_CONF_DIR={{atlas_hbase_conf_dir}} +export HBASE_CONF_DIR={{hbase_conf_dir}} # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. export METADATA_EXPANDED_WEBAPP_DIR={{expanded_war_dir}} http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml deleted file mode 100644 index 3c4826d..0000000 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml +++ /dev/null @@ -1,58 +0,0 @@ -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration supports_final="true"> - <property> - <name>hbase.rootdir</name> - <value>file://{{metadata_home}}/data/root</value> - </property> - <property> - <name>hbase.zookeeper.property.dataDir</name> - <value>{{metadata_home}}/data/zookeeper-data</value> - </property> - <property> - <name>hbase.master.info.port</name> - <value>61510</value> - </property> - <property> - <name>hbase.regionserver.info.port</name> - <value>61530</value> - </property> - <property> - <name>hbase.master.port</name> - <value>61500</value> - </property> - <property> - <name>hbase.regionserver.port</name> - <value>61520</value> - </property> - <property> - <name>hbase.zookeeper.property.clientPort</name> - <value>61581</value> - </property> - <property> - <name>hbase.zookeeper.property.peerport</name> - <value>61588</value> - </property> - <property> - <name>hbase.zookeeper.property.leaderport</name> - <value>61589</value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml index f4115f7..d0d73ad 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml @@ -59,11 +59,6 @@ <fileName>atlas-log4j.xml</fileName> <dictionaryName>atlas-log4j</dictionaryName> </configFile> - <configFile> - <type>xml</type> - <fileName>hbase-site.xml</fileName> - <dictionaryName>atlas-hbase-site</dictionaryName> - </configFile> </component> <component> @@ -93,11 +88,6 @@ <fileName>atlas-log4j.xml</fileName> <dictionaryName>atlas-log4j</dictionaryName> </configFile> - <configFile> - <type>xml</type> - <fileName>atlas-hbase-site.xml</fileName> - <dictionaryName>atlas-hbase-site</dictionaryName> - </configFile> </component> </components> @@ -118,15 +108,10 @@ <timeout>300</timeout> </commandScript> - <requiredServices> - <service>KAFKA</service> - </requiredServices> - <configuration-dependencies> <config-type>application-properties</config-type> <config-type>atlas-env</config-type> <config-type>atlas-log4j</config-type> - <config-type>atlas-hbase-site</config-type> </configuration-dependencies> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py index 6b045ca..6d70d2c 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py @@ -53,7 +53,7 @@ class AtlasClient(Script): def configure(self, env): import params env.set_params(params) - metadata() + metadata('client') def status(self, env): raise ClientComponentHasNoStatus() http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index e305138..6f287dc 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -18,17 +18,16 @@ limitations under the License. """ -from resource_management.core.resources.system import Directory, File -from resource_management.core.exceptions import Fail -from resource_management.core.logger import Logger +import random +from resource_management.core.resources.system import Directory, File from resource_management.core.source import StaticFile, InlineTemplate from resource_management.libraries.resources.properties_file import PropertiesFile -from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.libraries.functions.format import format from resource_management.libraries.resources.template_config import TemplateConfig +from resource_management.libraries.functions import solr_cloud_util -def metadata(): +def metadata(type='server'): import params Directory([params.pid_dir], @@ -55,22 +54,6 @@ def metadata(): create_parents = True ) - Directory(params.atlas_hbase_log_dir, - mode=0755, - cd_access='a', - owner=params.metadata_user, - group=params.user_group, - create_parents = True - ) - - Directory(params.atlas_hbase_data_dir, - mode=0755, - cd_access='a', - owner=params.metadata_user, - group=params.user_group, - create_parents = True - ) - Directory(params.data_dir, mode=0644, cd_access='a', @@ -126,16 +109,43 @@ def metadata(): content=StaticFile('policy-store.txt') ) - if params.atlas_has_embedded_hbase: - # hbase-site for embedded hbase used by Atlas - XmlConfig( "hbase-site.xml", - conf_dir = params.atlas_hbase_conf_dir, - configurations = params.config['configurations']['atlas-hbase-site'], - configuration_attributes=params.config['configuration_attributes']['atlas-hbase-site'], - owner = params.metadata_user, - group = params.user_group - ) + if type == 'server': + random_num = random.random() + + upload_conf_set('basic_configs', random_num) + + create_collection('vertex_index', 'basic_configs') + create_collection('edge_index', 'basic_configs') + create_collection('fulltext_index', 'basic_configs') if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user) + +def upload_conf_set(config_set, random_num): + import params + tmp_config_set_folder = format('{tmp_dir}/solr_config_{config_set}_{random_num}') + + solr_cloud_util.upload_configuration_to_zk( + zookeeper_quorum=params.zookeeper_quorum, + solr_znode=params.logsearch_solr_znode, + config_set_dir=format("{logsearch_solr_dir}/server/solr/configsets/{config_set}/conf"), + config_set=config_set, + tmp_config_set_dir=tmp_config_set_folder, + java64_home=params.java64_home, + user=params.metadata_user, + group=params.user_group) + +def create_collection(collection, config_set): + import params + + solr_cloud_util.create_collection( + zookeeper_quorum=params.zookeeper_quorum, + solr_znode=params.logsearch_solr_znode, + collection = collection, + config_set=config_set, + java64_home=params.java64_home, + user=params.metadata_user, + group=params.user_group, + shards=params.atlas_solr_shards, + replication_factor = params.logsearch_solr_replication_factor) http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index f172b79..3a9c614 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -29,6 +29,7 @@ import status_params # server configurations config = Script.get_config() stack_root = Script.get_stack_root() +tmp_dir = Script.get_tmp_dir() cluster_name = config['clusterName'] @@ -65,13 +66,6 @@ conf_file = status_params.conf_file atlas_login_credentials_file = os.path.join(conf_dir, "users-credentials.properties") atlas_policy_store_file = os.path.join(conf_dir, "policy-store.txt") -atlas_hbase_conf_dir = os.path.join(metadata_home, "hbase", "conf") -atlas_hbase_log_dir = os.path.join(metadata_home, "hbase", "logs") -atlas_hbase_data_dir = os.path.join(metadata_home, "data") -atlas_hbase_zk_port = default("/configurations/atlas-hbase-site/hbase.zookeeper.property.clientPort", None) - -atlas_has_embedded_hbase = default("/configurations/atlas-env/has_embedded_hbase", False) - # service locations hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOME' in os.environ else '/etc/hadoop/conf' @@ -144,6 +138,10 @@ if not len(kafka_broker_hosts) == 0: kafka_zookeeper_connect = default("/configurations/kafka-broker/zookeeper.connect", None) +# hbase +hbase_zookeeper_quorum = default('/configurations/hbase-site/hbase.zookeeper.quorum', None) +hbase_conf_dir = "/etc/hbase/conf" + # atlas HA atlas_hosts = sorted(default('/clusterHostInfo/atlas_server_hosts', [])) @@ -164,9 +162,17 @@ for host in atlas_hosts: id += 1 first_id = False +# logsearch solr +logsearch_solr_znode = default("/configurations/logsearch-solr-env/logsearch_solr_znode", None) +logsearch_solr_dir = '/usr/lib/ambari-logsearch-solr' +logsearch_solr_hosts = default("/clusterHostInfo/logsearch_solr_hosts", []) +logsearch_solr_replication_factor = 2 if len(logsearch_solr_hosts) > 1 else 1 +atlas_solr_shards = default("/configurations/atlas-env/atlas_solr-shards", 1) +has_logsearch_solr = len(logsearch_solr_hosts) > 0 + +# zookeeper zookeeper_hosts = config['clusterHostInfo']['zookeeper_hosts'] zookeeper_port = default('/configurations/zoo.cfg/clientPort', None) -logsearch_solr_znode = default("/configurations/logsearch-solr-env/logsearch_solr_znode", None) # get comma separated lists of zookeeper hosts from clusterHostInfo index = 0 @@ -174,7 +180,9 @@ zookeeper_quorum = "" solr_zookeeper_url = "" for host in zookeeper_hosts: - zookeeper_host = host + ":" + str(zookeeper_port) + zookeeper_host = host + if zookeeper_port is not None: + zookeeper_host = host + ":" + str(zookeeper_port) if logsearch_solr_znode is not None: solr_zookeeper_url += zookeeper_host + logsearch_solr_znode http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml index 0631b7d..1b4d0ea 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml @@ -49,7 +49,12 @@ </property> <property> <name>atlas.graph.storage.hostname</name> - <value>localhost:{{atlas_hbase_zk_port}}</value> + <value>{{hbase_zookeeper_quorum}}</value> + <description></description> + </property> + <property> + <name>atlas.audit.hbase.zookeeper.quorum</name> + <value>{{hbase_zookeeper_quorum}}</value> <description></description> </property> <property> @@ -64,6 +69,7 @@ </property> <property> <name>atlas.graph.index.search.solr.zookeeper-url</name> + <value>{{solr_zookeeper_url}}</value> <description>The ZooKeeper quorum setup for Solr as comma separated value.</description> </property> <property> @@ -82,6 +88,31 @@ <description>Path for the Atlas policy file.</description> </property> <property> + <name>atlas.kafka.data</name> + <value>{{metadata_home}}/data/kafka</value> + <description>The Kafka data directory.</description> + </property> + <property> + <name>atlas.kafka.bootstrap.servers</name> + <value>{{kafka_bootstrap_servers}}</value> + <description>Comma separated list of Kafka broker endpoints in host:port form</description> + </property> + <property> + <name>atlas.kafka.zookeeper.connect</name> + <value>{{kafka_zookeeper_connect}}</value> + <description>Comma separated list of servers forming Zookeeper quorum used by Kafka.</description> + </property> + <property> + <name>atlas.kafka.hook.group.id</name> + <value>atlas</value> + <description>Kafka group id for the hook topic.</description> + </property> + <property> + <name>atlas.kafka.entities.group.id</name> + <value>entities</value> + <description>Kafka group id for the entity topic.</description> + </property> + <property> <name>atlas.kafka.auto.commit.enable</name> <value>false</value> <description>Kafka auto commit setting for Atlas notifications.</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml index 2a5f777..e44872d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml @@ -80,16 +80,15 @@ </value-attributes> </property> <property> - <name>has_embedded_hbase</name> - <value>true</value> - <description>Indicates whether or not Atlas can install embedded Hbase.</description> - <value-attributes> - <read-only>true</read-only> - <overridable>false</overridable> - <visible>false</visible> - </value-attributes> + <name>atlas_solr_shards</name> + <value>1</value> + <description>The number of shards set for LogSearch SOLR.</description> + </property> + <property> + <name>atlas_solr_shards</name> + <value>1</value> + <description>The number of shards set for collections created in LogSearch SOLR.</description> </property> - <!-- metadata-env.sh --> <property> <name>content</name> @@ -122,7 +121,7 @@ export ATLAS_PID_DIR={{pid_dir}} # hbase conf dir - export HBASE_CONF_DIR={{atlas_hbase_conf_dir}} + export HBASE_CONF_DIR={{hbase_conf_dir}} # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. export ATLAS_EXPANDED_WEBAPP_DIR={{expanded_war_dir}} http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml index 15daeea..4890f00 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml @@ -25,12 +25,21 @@ <component> <name>ATLAS_SERVER</name> <cardinality>1+</cardinality> + <dependencies> + <dependency> + <name>LOGSEARCH/LOGSEARCH_SOLR_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> <configFile> <type>properties</type> <fileName>atlas-application.properties</fileName> <dictionaryName>application-properties</dictionaryName> </configFile> - </component> + </component> <component> <name>ATLAS_CLIENT</name> @@ -41,6 +50,11 @@ </configFile> </component> </components> + + <requiredServices> + <service>KAFKA</service> + </requiredServices> + </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 093c819..44407ca 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -51,6 +51,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): def getServiceConfigurationValidators(self): parentValidators = super(HDP25StackAdvisor, self).getServiceConfigurationValidators() childValidators = { + "ATLAS": {"application-properties": self.validateAtlasConfigurations}, "HIVE": {"hive-interactive-env": self.validateHiveInteractiveEnvConfigurations}, "YARN": {"yarn-site": self.validateYarnConfigurations}, "RANGER": {"ranger-tagsync-site": self.validateRangerTagsyncConfigurations} @@ -58,6 +59,44 @@ class HDP25StackAdvisor(HDP24StackAdvisor): self.mergeValidators(parentValidators, childValidators) return parentValidators + def validateAtlasConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + application_properties = getSiteProperties(configurations, "application-properties") + validationItems = [] + + if application_properties['atlas.graph.index.search.backend'] == 'solr5' and \ + not application_properties['atlas.graph.index.search.solr.zookeeper-url']: + validationItems.append({"config-name": "atlas.graph.index.search.solr.zookeeper-url", + "item": self.getErrorItem( + "If LOGSEARCH is not installed then the SOLR zookeeper url configuration must be specified.")}) + + if not application_properties['atlas.kafka.bootstrap.servers']: + validationItems.append({"config-name": "atlas.kafka.bootstrap.servers", + "item": self.getErrorItem( + "If KAFKA is not installed then the Kafka bootstrap servers configuration must be specified.")}) + + if not application_properties['atlas.kafka.zookeeper.connect']: + validationItems.append({"config-name": "atlas.kafka.zookeeper.connect", + "item": self.getErrorItem( + "If KAFKA is not installed then the Kafka zookeeper quorum configuration must be specified.")}) + + if application_properties['atlas.graph.storage.backend'] == 'hbase': + if not application_properties['atlas.graph.storage.hostname']: + validationItems.append({"config-name": "atlas.graph.storage.hostname", + "item": self.getErrorItem( + "If HBASE is not installed then the hbase zookeeper quorum configuration must be specified.")}) + elif application_properties['atlas.graph.storage.hostname'] == '{{hbase_zookeeper_quorum}}': + validationItems.append({"config-name": "atlas.graph.storage.hostname", + "item": self.getWarnItem( + "Note that Atlas is configured to use the HBASE instance being installed for this cluster.")}) + + if not application_properties['atlas.audit.hbase.zookeeper.quorum']: + validationItems.append({"config-name": "atlas.audit.hbase.zookeeper.quorum", + "item": self.getErrorItem( + "If HBASE is not installed then the audit hbase zookeeper quorum configuration must be specified.")}) + + validationProblems = self.toConfigurationValidationProblems(validationItems, "application-properties") + return validationProblems + def validateYarnConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): parentValidationProblems = super(HDP25StackAdvisor, self).validateYARNConfigurations(properties, recommendedDefaults, configurations, services, hosts) yarn_site_properties = getSiteProperties(configurations, "yarn-site") @@ -134,9 +173,24 @@ class HDP25StackAdvisor(HDP24StackAdvisor): servicesList = [service["StackServices"]["service_name"] for service in services["services"]] - include_logsearch = "LOGSEARCH" in servicesList - if include_logsearch and "logsearch-solr-env" in services["configurations"]: + if "LOGSEARCH" in servicesList: putAtlasApplicationProperty('atlas.graph.index.search.solr.zookeeper-url', '{{solr_zookeeper_url}}') + else: + putAtlasApplicationProperty('atlas.graph.index.search.solr.zookeeper-url', "") + + if "KAFKA" in servicesList: + putAtlasApplicationProperty('atlas.kafka.bootstrap.servers', '{{kafka_bootstrap_servers}}') + putAtlasApplicationProperty('atlas.kafka.zookeeper.connect', '{{kafka_zookeeper_connect}}') + else: + putAtlasApplicationProperty('atlas.kafka.bootstrap.servers', "") + putAtlasApplicationProperty('atlas.kafka.zookeeper.connect', "") + + if "HBASE" in servicesList: + putAtlasApplicationProperty('atlas.graph.storage.hostname', '{{hbase_zookeeper_quorum}}') + putAtlasApplicationProperty('atlas.audit.hbase.zookeeper.quorum', '{{hbase_zookeeper_quorum}}') + else: + putAtlasApplicationProperty('atlas.graph.storage.hostname', "") + putAtlasApplicationProperty('atlas.audit.hbase.zookeeper.quorum', "") def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts): super(HDP25StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts) http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py index 98fc678..faa7350 100644 --- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py +++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py @@ -52,20 +52,6 @@ class TestMetadataServer(RMFTestCase): cd_access='a', mode=0755 ) - self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/hbase/logs', - owner='atlas', - group='hadoop', - create_parents = True, - cd_access='a', - mode=0755 - ) - self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data', - owner='atlas', - group='hadoop', - create_parents = True, - cd_access='a', - mode=0755 - ) self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data', owner='atlas', group='hadoop', @@ -122,13 +108,13 @@ class TestMetadataServer(RMFTestCase): group='hadoop', mode=0644, ) - self.assertResourceCalled('XmlConfig', 'hbase-site.xml', - owner = 'atlas', - group = 'hadoop', - conf_dir = '/usr/hdp/current/atlas-server/hbase/conf', - configurations = self.getConfig()['configurations']['atlas-hbase-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['atlas-hbase-site'] - ) + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --download-config -d /tmp/solr_config_basic_configs_0.[0-9]* -cs basic_configs -rt 5 -i 10') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --upload-config -d /usr/lib/ambari-logsearch-solr/server/solr/configsets/basic_configs/conf -cs basic_configs -rt 5 -i 10') + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --create-collection -c vertex_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --create-collection -c edge_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --create-collection -c fulltext_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10') def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py", http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/test/python/stacks/2.3/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json index c8b418b..ef04248 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json @@ -171,6 +171,9 @@ "log.index.interval.bytes": "4096", "log.retention.hours": "168" }, + "logsearch-solr-env": { + "logsearch_solr_znode": "/logsearch" + }, "application-properties": { "atlas.cluster.name" : "c2", "atlas.graph.storage.backend": "berkeleyje", @@ -210,8 +213,7 @@ "metadata_log_dir": "/var/log/atlas", "metadata_data_dir": "/var/lib/atlas/data", "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp", - "metadata_conf_file": "application.properties", - "has_embedded_hbase":"true" + "metadata_conf_file": "application.properties" }, "atlas-log4j": { "content": "<property><name>content</name><description>Custom log4j.properties</description><value></value></property>", http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/test/python/stacks/2.3/configs/secure.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/secure.json b/ambari-server/src/test/python/stacks/2.3/configs/secure.json index 7ebcedf..166fd6f 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/secure.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/secure.json @@ -162,6 +162,9 @@ "log.index.interval.bytes": "4096", "log.retention.hours": "168" }, + "logsearch-solr-env": { + "logsearch_solr_znode": "/logsearch" + }, "application-properties": { "atlas.cluster.name" : "c2", "atlas.graph.storage.backend": "berkeleyje", @@ -202,8 +205,7 @@ "metadata_log_dir": "/var/log/atlas", "metadata_data_dir": "/var/lib/atlas/data", "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp", - "metadata_conf_file": "application.properties", - "has_embedded_hbase":"true" + "metadata_conf_file": "application.properties" }, "atlas-log4j": { "content": "<property><name>content</name><description>Custom log4j.properties</description><value></value></property>", http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py index 9d0f00c..6839dee 100644 --- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py +++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py @@ -49,20 +49,6 @@ class TestAtlasServer(RMFTestCase): cd_access='a', mode=0755 ) - self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/hbase/logs', - owner='atlas', - group='hadoop', - create_parents = True, - cd_access='a', - mode=0755 - ) - self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data', - owner='atlas', - group='hadoop', - create_parents = True, - cd_access='a', - mode=0755 - ) self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data', owner='atlas', group='hadoop', @@ -120,13 +106,14 @@ class TestAtlasServer(RMFTestCase): group='hadoop', mode=0644, ) - self.assertResourceCalled('XmlConfig', 'hbase-site.xml', - owner = 'atlas', - group = 'hadoop', - conf_dir = '/usr/hdp/current/atlas-server/hbase/conf', - configurations = self.getConfig()['configurations']['atlas-hbase-site'], - configuration_attributes = self.getConfig()['configuration_attributes']['atlas-hbase-site'] - ) + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --download-config -d /tmp/solr_config_basic_configs_0.[0-9]* -cs basic_configs -rt 5 -i 10') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --upload-config -d /usr/lib/ambari-logsearch-solr/server/solr/configsets/basic_configs/conf -cs basic_configs -rt 5 -i 10') + + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --create-collection -c vertex_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --create-collection -c edge_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10') + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --create-collection -c fulltext_index -cs basic_configs -s 1 -r 1 -m 1 -rt 5 -i 10') + def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py", http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index a7e1293..01064a6 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -4772,7 +4772,11 @@ class TestHDP25StackAdvisor(TestCase): configurations = { "application-properties": { "properties": { - "atlas.graph.index.search.solr.zookeeper-url": "" + "atlas.graph.index.search.solr.zookeeper-url": "", + "atlas.audit.hbase.zookeeper.quorum": "", + "atlas.graph.storage.hostname": "", + "atlas.kafka.bootstrap.servers": "", + "atlas.kafka.zookeeper.connect": "" } }, "logsearch-solr-env": { @@ -4793,6 +4797,10 @@ class TestHDP25StackAdvisor(TestCase): 'application-properties': { 'properties': { 'atlas.graph.index.search.solr.zookeeper-url': '{{solr_zookeeper_url}}', + "atlas.audit.hbase.zookeeper.quorum": "", + "atlas.graph.storage.hostname": "", + "atlas.kafka.bootstrap.servers": "", + "atlas.kafka.zookeeper.connect": "" } }, "logsearch-solr-env": { @@ -4831,7 +4839,11 @@ class TestHDP25StackAdvisor(TestCase): "configurations": { "application-properties": { "properties": { - "atlas.graph.index.search.solr.zookeeper-url": "" + "atlas.graph.index.search.solr.zookeeper-url": "", + "atlas.audit.hbase.zookeeper.quorum": "", + "atlas.graph.storage.hostname": "", + "atlas.kafka.bootstrap.servers": "", + "atlas.kafka.zookeeper.connect": "" } }, "logsearch-solr-env": { http://git-wip-us.apache.org/repos/asf/ambari/blob/14e4956f/ambari-server/src/test/python/stacks/2.5/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/default.json b/ambari-server/src/test/python/stacks/2.5/configs/default.json index e4a97f6..0077280 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/default.json @@ -171,6 +171,9 @@ "log.index.interval.bytes": "4096", "log.retention.hours": "168" }, + "logsearch-solr-env": { + "logsearch_solr_znode": "/logsearch" + }, "application-properties": { "atlas.cluster.name" : "c2", "atlas.graph.storage.backend": "berkeleyje", @@ -210,8 +213,7 @@ "metadata_log_dir": "/var/log/atlas", "metadata_data_dir": "/var/lib/atlas/data", "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp", - "metadata_conf_file": "atlas-application.properties", - "has_embedded_hbase":"true" + "metadata_conf_file": "atlas-application.properties" }, "atlas-log4j": { "content": "<property><name>content</name><description>Custom log4j.properties</description><value></value></property>",
