AMBARI-18818 Config changes for Atlas to Support KnoxSSO Authentication (mugdha)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5454bd61 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5454bd61 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5454bd61 Branch: refs/heads/branch-dev-patch-upgrade Commit: 5454bd612774d7f17704adaff24cbfa73c6a4f77 Parents: 8c3ad1d Author: Mugdha Varadkar <[email protected]> Authored: Tue Nov 15 11:30:25 2016 +0530 Committer: Mugdha Varadkar <[email protected]> Committed: Wed Nov 16 10:23:32 2016 +0530 ---------------------------------------------------------------------- .../configuration/application-properties.xml | 11 + .../configuration/application-properties.xml | 95 +++ .../ATLAS/configuration/atlas-log4j.xml | 118 +++ .../stacks/HDP/2.6/services/ATLAS/metainfo.xml | 10 +- .../services/ATLAS/themes/theme_version_2.json | 845 +++++++++++++++++++ .../stacks/HDP/2.6/services/stack_advisor.py | 381 +++++---- .../stacks/2.6/common/test_stack_advisor.py | 97 +++ 7 files changed, 1374 insertions(+), 183 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml index 95715b3..fda16b2 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml @@ -77,18 +77,29 @@ <value>false</value> <description>Indicates whether or not Kerberos is enabled.</description> <on-ambari-upgrade add="false"/> + <value-attributes> + <type>boolean</type> + </value-attributes> </property> <property> <name>atlas.authentication.method.file</name> + <display-name>Enable File Authentication</display-name> <value>true</value> <description>Indicates whether or not file based authentication is enabled.</description> <on-ambari-upgrade add="false"/> + <value-attributes> + <type>boolean</type> + </value-attributes> </property> <property> <name>atlas.authentication.method.ldap</name> + <display-name>Enable LDAP Authentication</display-name> <value>false</value> <description>Indicates whether or not LDAP authentication is enabled.</description> <on-ambari-upgrade add="false"/> + <value-attributes> + <type>boolean</type> + </value-attributes> </property> <property> <name>atlas.authentication.method.file.filename</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml new file mode 100644 index 0000000..0d6ee38 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml @@ -0,0 +1,95 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false"> + + <!-- Atlas knox sso properties --> + <property> + <name>atlas.sso.knox.enabled</name> + <display-name>Enable Atlas Knox SSO</display-name> + <value>false</value> + <description/> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>atlas.sso.knox.providerurl</name> + <value/> + <description/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <depends-on> + <property> + <type>gateway-site</type> + <name>gateway.port</name> + </property> + </depends-on> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>atlas.sso.knox.publicKey</name> + <value/> + <description/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>atlas.sso.knox.browser.useragent</name> + <value/> + <description/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>atlas.authentication.method.ldap.type</name> + <display-name>LDAP Authentication Type</display-name> + <value>ldap</value> + <description>The LDAP type (ldap, ad, or none).</description> + <value-attributes> + <overridable>false</overridable> + <type>value-list</type> + <entries> + <entry> + <value>ldap</value> + <label>LDAP</label> + </entry> + <entry> + <value>ad</value> + <label>AD</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/atlas-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/atlas-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/atlas-log4j.xml new file mode 100644 index 0000000..aebf241 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/atlas-log4j.xml @@ -0,0 +1,118 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <property> + <name>content</name> + <display-name>atlas-log4j template</display-name> + <description>Custom log4j.properties</description> + <value><![CDATA[<?xml version="1.0" encoding="UTF-8" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/> + </layout> + </appender> + + <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="File" value="{{log_dir}}/application.log"/> + <param name="Append" value="true"/> + <param name="Threshold" value="info"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/> + </layout> + </appender> + + <appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="File" value="{{log_dir}}/audit.log"/> + <param name="Append" value="true"/> + <param name="Threshold" value="info"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %x %m%n"/> + </layout> + </appender> + + <logger name="org.apache.atlas" additivity="false"> + <level value="{{atlas_log_level}}"/> + <appender-ref ref="FILE"/> + </logger> + + + <logger name="com.thinkaurelius.titan" additivity="false"> + <level value="info"/> + <appender-ref ref="FILE"/> + </logger> + + <logger name="org.elasticsearch" additivity="false"> + <level value="info"/> + <appender-ref ref="FILE"/> + </logger> + + <logger name="org.apache.lucene" additivity="false"> + <level value="info"/> + <appender-ref ref="FILE"/> + </logger> + + <logger name="com.google" additivity="false"> + <level value="info"/> + <appender-ref ref="FILE"/> + </logger> + + <logger name="AUDIT" additivity="false"> + <level value="{{audit_log_level}}"/> + <appender-ref ref="AUDIT"/> + </logger> + + <root> + <priority value="info"/> + <appender-ref ref="FILE"/> + </root> + +</log4j:configuration> + ]]></value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml index 92db1f9..eaa5e21 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml @@ -20,7 +20,15 @@ <services> <service> <name>ATLAS</name> - <version>0.7.0.2.5</version> + <version>0.8.0.2.6</version> + + <themes> + <theme> + <fileName>theme_version_2.json</fileName> + <default>true</default> + </theme> + </themes> + </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/themes/theme_version_2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/themes/theme_version_2.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/themes/theme_version_2.json new file mode 100644 index 0000000..f72ea4b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/themes/theme_version_2.json @@ -0,0 +1,845 @@ +{ + "name": "default", + "description": "Default theme for Atlas service", + "configuration": { + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "authentication_settings", + "display-name": "Authentication", + "layout": { + "tab-columns": "1", + "tab-rows": "4", + "sections": [ + { + "name": "section-authentication-type", + "display-name": "Authentication Methods", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-authentication-type", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "subsection-authentication-file", + "display-name": "File", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-authentication-file", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-authentication-ldap", + "display-name": "LDAP/AD", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-authentication-ldap", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + } + ] + }, + { + "name": "section-atlas-sso", + "display-name": "Atlas Knox SSO", + "row-index": "3", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-atlas-sso", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "application-properties/atlas.authentication.method.file", + "subsection-name": "subsection-authentication-type" + }, + { + "config": "application-properties/atlas.authentication.method.file.filename", + "subsection-name": "subsection-authentication-file", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.file" + ], + "if": "${application-properties/atlas.authentication.method.file}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap", + "subsection-name": "subsection-authentication-type" + }, + { + "config": "application-properties/atlas.authentication.method.ldap.type", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.url", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.userDNpattern", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap} ", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.groupSearchBase", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.groupSearchFilter", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.groupRoleAttribute", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.base.dn", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.bind.dn", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.bind.password", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.referral", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.user.searchfilter", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.default.role", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.url", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.domain", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.base.dn", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.bind.dn", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.bind.password", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.referral", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.user.searchfilter", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.default.role", + "subsection-name": "subsection-authentication-ldap", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.authentication.method.ldap.type", + "application-properties/atlas.authentication.method.ldap" + ], + "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.sso.enabled", + "subsection-name": "subsection-authentication-type" + }, + { + "config": "application-properties/atlas.sso.providerurl", + "subsection-name": "subsection-atlas-sso", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.sso.enabled" + ], + "if": "${application-properties/atlas.sso.enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.sso.publicKey", + "subsection-name": "subsection-atlas-sso", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.sso.enabled" + ], + "if": "${application-properties/atlas.sso.enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "application-properties/atlas.sso.browser.useragent", + "subsection-name": "subsection-atlas-sso", + "depends-on": [ + { + "configs":[ + "application-properties/atlas.sso.enabled" + ], + "if": "${application-properties/atlas.sso.enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + } + ] + }, + "widgets": [ + { + "config": "application-properties/atlas.authentication.method.file", + "widget": { + "type": "checkbox" + } + }, + { + "config": "application-properties/atlas.authentication.method.file.filename", + "widget": { + "type": "text-field" + } + }, + { + "config": "application-properties/atlas.authentication.method.ldap", + "widget": { + "type": "checkbox" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.url", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.userDNpattern", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.groupSearchBase", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.groupSearchFilter", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.groupRoleAttribute", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.base.dn", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.bind.dn", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.bind.password", + "widget":{ + "type":"password" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.referral", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.user.searchfilter", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.default.role", + "widget":{ + "type":"text-field" + } + }, + { + "config": "application-properties/atlas.authentication.method.ldap.type", + "widget": { + "type": "combo" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.ad.url", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.ad.domain", + "widget":{ + "type":"text-field" + } + }, + { + "config":"application-properties/atlas.authentication.method.ldap.ad.base.dn", + "widget":{ + "type":"text-field" + } + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.bind.dn", + "widget": { + "type": "text-field" + } + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.bind.password", + "widget": { + "type": "password" + } + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.referral", + "widget": { + "type": "text-field" + } + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.user.searchfilter", + "widget": { + "type": "text-field" + } + }, + { + "config": "application-properties/atlas.authentication.method.ldap.ad.default.role", + "widget": { + "type": "text-field" + } + }, + { + "config": "application-properties/atlas.sso.enabled", + "widget": { + "type":"checkbox" + } + }, + { + "config": "application-properties/atlas.sso.providerurl", + "widget": { + "type": "text-field" + } + }, + { + "config": "application-properties/atlas.sso.publicKey", + "widget": { + "type": "text-field" + } + }, + { + "config": "application-properties/atlas.sso.browser.useragent", + "widget": { + "type": "text-field" + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index 49dd086..718da2c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -22,185 +22,202 @@ from resource_management.libraries.functions import format class HDP26StackAdvisor(HDP25StackAdvisor): - def __init__(self): - super(HDP26StackAdvisor, self).__init__() - Logger.initialize_logger() - - def getServiceConfigurationRecommenderDict(self): - parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict() - childRecommendConfDict = { - "DRUID": self.recommendDruidConfigurations - } - parentRecommendConfDict.update(childRecommendConfDict) - return parentRecommendConfDict - - def recommendDruidConfigurations(self, configurations, clusterData, services, hosts): - - componentsListList = [service["components"] for service in services["services"]] - componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] - servicesList = [service["StackServices"]["service_name"] for service in services["services"]] - putCommonProperty = self.putProperty(configurations, "druid-common", services) - - putCommonProperty('druid.zk.service.host', self.getZKHostPortString(services)) - self.recommendDruidMaxMemoryLimitConfigurations(configurations, clusterData, services, hosts) - - # recommending the metadata storage uri - database_name = services['configurations']["druid-common"]["properties"]["database_name"] - metastore_hostname = services['configurations']["druid-common"]["properties"]["metastore_hostname"] - database_type = services['configurations']["druid-common"]["properties"]["druid.metadata.storage.type"] - metadata_storage_port = "1527" - mysql_extension_name = "io.druid.extensions:mysql-metadata-storage" - mysql_module_name = "mysql-metadata-storage" - postgres_module_name = "postgresql-metadata-storage" - extensions_load_list = services['configurations']['druid-common']['properties']['druid.extensions.loadList'] - extensions_pull_list = services['configurations']['druid-common']['properties']['druid.extensions.pullList'] - putDruidCommonProperty = self.putProperty(configurations, "druid-common", services) - - extensions_pull_list = self.removeFromList(extensions_pull_list, mysql_extension_name) - extensions_load_list = self.removeFromList(extensions_load_list, mysql_module_name) - extensions_load_list = self.removeFromList(extensions_load_list, postgres_module_name) - - if database_type == 'mysql': - metadata_storage_port = "3306" - extensions_pull_list = self.addToList(extensions_pull_list, mysql_extension_name) - extensions_load_list = self.addToList(extensions_load_list, mysql_module_name) - - if database_type == 'postgres': - extensions_load_list = self.addToList(extensions_load_list, postgres_module_name) - metadata_storage_port = "5432" - - putDruidCommonProperty('druid.metadata.storage.connector.port', metadata_storage_port) - putDruidCommonProperty('druid.metadata.storage.connector.connectURI', - self.getMetadataConnectionString(database_type).format(metastore_hostname, database_name, - metadata_storage_port)) - # HDFS is installed - if "HDFS" in servicesList and "hdfs-site" in services["configurations"]: - # recommend HDFS as default deep storage - extensions_load_list = self.addToList(extensions_load_list, "druid-hdfs-storage") - putCommonProperty("druid.storage.type", "hdfs") - putCommonProperty("druid.storage.storageDirectory", "/user/druid/data") - # configure indexer logs configs - putCommonProperty("druid.indexer.logs.type", "hdfs") - putCommonProperty("druid.indexer.logs.directory", "/user/druid/logs") - - if "KAFKA" in servicesList: - extensions_load_list = self.addToList(extensions_load_list, "druid-kafka-indexing-service") - - putCommonProperty('druid.extensions.loadList', extensions_load_list) - putCommonProperty('druid.extensions.pullList', extensions_pull_list) - - # JVM Configs go to env properties - putEnvProperty = self.putProperty(configurations, "druid-env", services) - - # processing thread pool Config - for component in ['DRUID_HISTORICAL', 'DRUID_BROKER']: - component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts) - nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component] - putComponentProperty = self.putProperty(configurations, format("druid-{nodeType}"), services) - if (component_hosts is not None and len(component_hosts) > 0): - totalAvailableCpu = self.getMinCpu(component_hosts) - processingThreads = 1 - if totalAvailableCpu > 1: - processingThreads = totalAvailableCpu - 1 - putComponentProperty('druid.processing.numThreads', processingThreads) - putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30) - - def getMetadataConnectionString(self, database_type): - driverDict = { - 'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true', - 'derby': 'jdbc:derby://{0}:{2}/{1};create=true', - 'postgres': 'jdbc:postgresql://{0}:{2}/{1}' - } - return driverDict.get(database_type.lower()) - - def addToList(self, json_list, word): - desr_list = json.loads(json_list) - if word not in desr_list: - desr_list.append(word) - return json.dumps(desr_list) - - def removeFromList(self, json_list, word): - desr_list = json.loads(json_list) - if word in desr_list: - desr_list.remove(word) - return json.dumps(desr_list) - - def recommendDruidMaxMemoryLimitConfigurations(self, configurations, clusterData, services, hosts): - putEnvPropertyAttribute = self.putPropertyAttribute(configurations, "druid-env") - for component in ["DRUID_HISTORICAL", "DRUID_MIDDLEMANAGER", "DRUID_BROKER", "DRUID_OVERLORD", - "DRUID_COORDINATOR"]: - component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts) - if component_hosts is not None and len(component_hosts) > 0: - totalAvailableMem = self.getMinMemory(component_hosts) / 1024 # In MB - nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component] - putEnvPropertyAttribute(format('druid.{nodeType}.jvm.heap.memory'), 'maximum', - max(totalAvailableMem, 1024)) - - DRUID_COMPONENT_NODE_TYPE_MAP = { - 'DRUID_BROKER': 'broker', - 'DRUID_COORDINATOR': 'coordinator', - 'DRUID_HISTORICAL': 'historical', - 'DRUID_MIDDLEMANAGER': 'middlemanager', - 'DRUID_OVERLORD': 'overlord', - 'DRUID_ROUTER': 'router' - } - - def getMinMemory(self, component_hosts): - min_ram_kb = 1073741824 # 1 TB - for host in component_hosts: - ram_kb = host['Hosts']['total_mem'] - min_ram_kb = min(min_ram_kb, ram_kb) - return min_ram_kb - - def getMinCpu(self, component_hosts): - min_cpu = 256 - for host in component_hosts: - cpu_count = host['Hosts']['cpu_count'] - min_cpu = min(min_cpu, cpu_count) - return min_cpu - - def getServiceConfigurationValidators(self): - parentValidators = super(HDP26StackAdvisor, self).getServiceConfigurationValidators() - childValidators = { - "DRUID": {"druid-env": self.validateDruidEnvConfigurations, - "druid-historical": self.validateDruidHistoricalConfigurations, - "druid-broker": self.validateDruidBrokerConfigurations} - } - self.mergeValidators(parentValidators, childValidators) - return parentValidators - - def validateDruidEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): - validationItems = [] - # Minimum Direct memory Validation - envProperties = services['configurations']['druid-env']['properties'] - for nodeType in ['broker', 'historical']: - properties = services['configurations'][format('druid-{nodeType}')]['properties'] - intermediateBufferSize = int(properties['druid.processing.buffer.sizeBytes']) / (1024 * 1024) # In MBs - processingThreads = int(properties['druid.processing.numThreads']) - directMemory = int(envProperties[format('druid.{nodeType}.jvm.direct.memory')]) - if directMemory < (processingThreads + 1) * intermediateBufferSize: - validationItems.extend( - {"config-name": format("druid.{nodeType}.jvm.direct.memory"), "item": self.getErrorItem( - format( - "Not enough direct memory available for {nodeType} Node." - "Please adjust druid.{nodeType}.jvm.direct.memory, druid.processing.buffer.sizeBytes, druid.processing.numThreads" - ) - ) - }) - return self.toConfigurationValidationProblems(validationItems, "druid-env") - - def validateDruidHistoricalConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): - validationItems = [ - {"config-name": "druid.processing.numThreads", - "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, - "druid.processing.numThreads")} - ] - return self.toConfigurationValidationProblems(validationItems, "druid-historical") - - def validateDruidBrokerConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): - validationItems = [ - {"config-name": "druid.processing.numThreads", - "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, - "druid.processing.numThreads")} - ] - return self.toConfigurationValidationProblems(validationItems, "druid-broker") + def __init__(self): + super(HDP26StackAdvisor, self).__init__() + Logger.initialize_logger() + + def getServiceConfigurationRecommenderDict(self): + parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict() + childRecommendConfDict = { + "DRUID": self.recommendDruidConfigurations, + "ATLAS": self.recommendAtlasConfigurations + } + parentRecommendConfDict.update(childRecommendConfDict) + return parentRecommendConfDict + + def recommendAtlasConfigurations(self, configurations, clusterData, services, hosts): + super(HDP26StackAdvisor, self).recommendAtlasConfigurations(configurations, clusterData, services, hosts) + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putAtlasApplicationProperty = self.putProperty(configurations, "application-properties", services) + + knox_host = 'localhost' + knox_port = '8443' + if 'KNOX' in servicesList: + knox_hosts = self.getComponentHostNames(services, "KNOX", "KNOX_GATEWAY") + if len(knox_hosts) > 0: + knox_hosts.sort() + knox_host = knox_hosts[0] + if 'gateway-site' in services['configurations'] and 'gateway.port' in services['configurations']["gateway-site"]["properties"]: + knox_port = services['configurations']["gateway-site"]["properties"]['gateway.port'] + putAtlasApplicationProperty('atlas.sso.providerurl', 'https://{0}:{1}/gateway/knoxsso/api/v1/websso'.format(knox_host, knox_port)) + + def recommendDruidConfigurations(self, configurations, clusterData, services, hosts): + + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putCommonProperty = self.putProperty(configurations, "druid-common", services) + + putCommonProperty('druid.zk.service.host', self.getZKHostPortString(services)) + self.recommendDruidMaxMemoryLimitConfigurations(configurations, clusterData, services, hosts) + + # recommending the metadata storage uri + database_name = services['configurations']["druid-common"]["properties"]["database_name"] + metastore_hostname = services['configurations']["druid-common"]["properties"]["metastore_hostname"] + database_type = services['configurations']["druid-common"]["properties"]["druid.metadata.storage.type"] + metadata_storage_port = "1527" + mysql_extension_name = "io.druid.extensions:mysql-metadata-storage" + mysql_module_name = "mysql-metadata-storage" + postgres_module_name = "postgresql-metadata-storage" + extensions_load_list = services['configurations']['druid-common']['properties']['druid.extensions.loadList'] + extensions_pull_list = services['configurations']['druid-common']['properties']['druid.extensions.pullList'] + putDruidCommonProperty = self.putProperty(configurations, "druid-common", services) + + extensions_pull_list = self.removeFromList(extensions_pull_list, mysql_extension_name) + extensions_load_list = self.removeFromList(extensions_load_list, mysql_module_name) + extensions_load_list = self.removeFromList(extensions_load_list, postgres_module_name) + + if database_type == 'mysql': + metadata_storage_port = "3306" + extensions_pull_list = self.addToList(extensions_pull_list, mysql_extension_name) + extensions_load_list = self.addToList(extensions_load_list, mysql_module_name) + + if database_type == 'postgres': + extensions_load_list = self.addToList(extensions_load_list, postgres_module_name) + metadata_storage_port = "5432" + + putDruidCommonProperty('druid.metadata.storage.connector.port', metadata_storage_port) + putDruidCommonProperty('druid.metadata.storage.connector.connectURI', + self.getMetadataConnectionString(database_type).format(metastore_hostname, database_name, + metadata_storage_port)) + # HDFS is installed + if "HDFS" in servicesList and "hdfs-site" in services["configurations"]: + # recommend HDFS as default deep storage + extensions_load_list = self.addToList(extensions_load_list, "druid-hdfs-storage") + putCommonProperty("druid.storage.type", "hdfs") + putCommonProperty("druid.storage.storageDirectory", "/user/druid/data") + # configure indexer logs configs + putCommonProperty("druid.indexer.logs.type", "hdfs") + putCommonProperty("druid.indexer.logs.directory", "/user/druid/logs") + + if "KAFKA" in servicesList: + extensions_load_list = self.addToList(extensions_load_list, "druid-kafka-indexing-service") + + putCommonProperty('druid.extensions.loadList', extensions_load_list) + putCommonProperty('druid.extensions.pullList', extensions_pull_list) + + # JVM Configs go to env properties + putEnvProperty = self.putProperty(configurations, "druid-env", services) + + # processing thread pool Config + for component in ['DRUID_HISTORICAL', 'DRUID_BROKER']: + component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts) + nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component] + putComponentProperty = self.putProperty(configurations, format("druid-{nodeType}"), services) + if (component_hosts is not None and len(component_hosts) > 0): + totalAvailableCpu = self.getMinCpu(component_hosts) + processingThreads = 1 + if totalAvailableCpu > 1: + processingThreads = totalAvailableCpu - 1 + putComponentProperty('druid.processing.numThreads', processingThreads) + putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30) + + def getMetadataConnectionString(self, database_type): + driverDict = { + 'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true', + 'derby': 'jdbc:derby://{0}:{2}/{1};create=true', + 'postgres': 'jdbc:postgresql://{0}:{2}/{1}' + } + return driverDict.get(database_type.lower()) + + def addToList(self, json_list, word): + desr_list = json.loads(json_list) + if word not in desr_list: + desr_list.append(word) + return json.dumps(desr_list) + + def removeFromList(self, json_list, word): + desr_list = json.loads(json_list) + if word in desr_list: + desr_list.remove(word) + return json.dumps(desr_list) + + def recommendDruidMaxMemoryLimitConfigurations(self, configurations, clusterData, services, hosts): + putEnvPropertyAttribute = self.putPropertyAttribute(configurations, "druid-env") + for component in ["DRUID_HISTORICAL", "DRUID_MIDDLEMANAGER", "DRUID_BROKER", "DRUID_OVERLORD", + "DRUID_COORDINATOR"]: + component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts) + if component_hosts is not None and len(component_hosts) > 0: + totalAvailableMem = self.getMinMemory(component_hosts) / 1024 # In MB + nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component] + putEnvPropertyAttribute(format('druid.{nodeType}.jvm.heap.memory'), 'maximum', + max(totalAvailableMem, 1024)) + + DRUID_COMPONENT_NODE_TYPE_MAP = { + 'DRUID_BROKER': 'broker', + 'DRUID_COORDINATOR': 'coordinator', + 'DRUID_HISTORICAL': 'historical', + 'DRUID_MIDDLEMANAGER': 'middlemanager', + 'DRUID_OVERLORD': 'overlord', + 'DRUID_ROUTER': 'router' + } + + def getMinMemory(self, component_hosts): + min_ram_kb = 1073741824 # 1 TB + for host in component_hosts: + ram_kb = host['Hosts']['total_mem'] + min_ram_kb = min(min_ram_kb, ram_kb) + return min_ram_kb + + def getMinCpu(self, component_hosts): + min_cpu = 256 + for host in component_hosts: + cpu_count = host['Hosts']['cpu_count'] + min_cpu = min(min_cpu, cpu_count) + return min_cpu + + def getServiceConfigurationValidators(self): + parentValidators = super(HDP26StackAdvisor, self).getServiceConfigurationValidators() + childValidators = { + "DRUID": {"druid-env": self.validateDruidEnvConfigurations, + "druid-historical": self.validateDruidHistoricalConfigurations, + "druid-broker": self.validateDruidBrokerConfigurations} + } + self.mergeValidators(parentValidators, childValidators) + return parentValidators + + def validateDruidEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + validationItems = [] + # Minimum Direct memory Validation + envProperties = services['configurations']['druid-env']['properties'] + for nodeType in ['broker', 'historical']: + properties = services['configurations'][format('druid-{nodeType}')]['properties'] + intermediateBufferSize = int(properties['druid.processing.buffer.sizeBytes']) / (1024 * 1024) # In MBs + processingThreads = int(properties['druid.processing.numThreads']) + directMemory = int(envProperties[format('druid.{nodeType}.jvm.direct.memory')]) + if directMemory < (processingThreads + 1) * intermediateBufferSize: + validationItems.extend( + {"config-name": format("druid.{nodeType}.jvm.direct.memory"), "item": self.getErrorItem( + format( + "Not enough direct memory available for {nodeType} Node." + "Please adjust druid.{nodeType}.jvm.direct.memory, druid.processing.buffer.sizeBytes, druid.processing.numThreads" + ) + ) + }) + return self.toConfigurationValidationProblems(validationItems, "druid-env") + + def validateDruidHistoricalConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + validationItems = [ + {"config-name": "druid.processing.numThreads", + "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, + "druid.processing.numThreads")} + ] + return self.toConfigurationValidationProblems(validationItems, "druid-historical") + + def validateDruidBrokerConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + validationItems = [ + {"config-name": "druid.processing.numThreads", + "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, + "druid.processing.numThreads")} + ] + return self.toConfigurationValidationProblems(validationItems, "druid-broker") http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py index 8227d69..a7a45f8 100644 --- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py @@ -568,6 +568,103 @@ class TestHDP26StackAdvisor(TestCase): ) + def test_recommendAtlasConfigurations(self): + configurations = { + "application-properties": { + "properties": { + "atlas.sso.providerurl": "", + "atlas.graph.index.search.solr.zookeeper-url": "", + "atlas.audit.hbase.zookeeper.quorum": "", + "atlas.graph.storage.hostname": "", + "atlas.kafka.bootstrap.servers": "", + "atlas.kafka.zookeeper.connect": "", + "atlas.authorizer.impl": "simple" + } + }, + "infra-solr-env": { + "properties": { + "infra_solr_znode": "/infra-solr" + } + }, + "ranger-atlas-plugin-properties": { + "properties": { + "ranger-atlas-plugin-enabled":"No" + } + }, + "atlas-env": { + "properties": { + "atlas_server_max_new_size": "600", + "atlas_server_xmx": "2048" + } + } + } + + clusterData = {} + + expected = { + "application-properties": { + "properties": { + "atlas.sso.providerurl": "https://c6401.ambari.apache.org:8443/gateway/knoxsso/api/v1/websso";, + "atlas.graph.index.search.solr.zookeeper-url": "", + "atlas.audit.hbase.zookeeper.quorum": "", + "atlas.graph.storage.hostname": "", + "atlas.kafka.bootstrap.servers": "", + "atlas.kafka.zookeeper.connect": "", + "atlas.authorizer.impl": "simple" + } + }, + "infra-solr-env": { + "properties": { + "infra_solr_znode": "/infra-solr" + } + }, + "ranger-atlas-plugin-properties": { + "properties": { + "ranger-atlas-plugin-enabled":"No" + } + }, + "atlas-env": { + "properties": { + "atlas_server_max_new_size": "600", + "atlas_server_xmx": "2048" + } + } + } + + services = { + "services": [ + { + "href": "/api/v1/stacks/HDP/versions/2.6/services/KNOX", + "StackServices": { + "service_name": "KNOX", + "service_version": "0.9.0.2.5", + "stack_name": "HDP", + "stack_version": "2.6" + }, + "components": [ + { + "href": "/api/v1/stacks/HDP/versions/2.6/services/KNOX/components/KNOX_GATEWAY", + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1+", + "component_category": "MASTER", + "component_name": "KNOX_GATEWAY", + "display_name": "Knox Gateway", + "is_client": "false", + "is_master": "true", + "hostnames": ["c6401.ambari.apache.org"] + }, + "dependencies": [] + } + ] + } + ], + "configurations": configurations + } + + self.stackAdvisor.recommendAtlasConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations, expected) + def load_json(self, filename): file = os.path.join(self.testDirectory, filename) with open(file, 'rb') as f:
