Repository: ambari Updated Branches: refs/heads/branch-2.1 989ced8be -> d304151ba
AMBARI-12875. BE: Modify Hive stack definition to support SQL Anywhere.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d304151b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d304151b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d304151b Branch: refs/heads/branch-2.1 Commit: d304151ba91859dc6248d9f939831e68d8ee8950 Parents: 989ced8 Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Wed Aug 26 14:32:01 2015 +0300 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Wed Aug 26 14:32:54 2015 +0300 ---------------------------------------------------------------------- .../HIVE/0.12.0.2.0/configuration/hive-env.xml | 3 - .../services/HIVE/configuration/hive-env.xml | 3 - .../services/HIVE/configuration/hive-env.xml | 71 ++++++++++++++++++++ .../services/HIVE/configuration/hive-site.xml | 12 ++++ .../stacks/HDP/2.3/services/stack_advisor.py | 34 +++++++++- .../stacks/2.3/common/test_stack_advisor.py | 3 +- 6 files changed, 118 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d304151b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml index 21c52e1..784f06b 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml @@ -153,9 +153,6 @@ else fi export METASTORE_PORT={{hive_metastore_port}} -{% if sqla_db_used %} -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}" -{% endif %} </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/d304151b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml index 6420333..ec0381a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml @@ -72,9 +72,6 @@ fi export METASTORE_PORT={{hive_metastore_port}} -{% if sqla_db_used %} -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}" -{% endif %} </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/d304151b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml new file mode 100644 index 0000000..b7b3127 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml @@ -0,0 +1,71 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + + <!-- hive-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for hive-env.sh file</description> + <value> + if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi + fi + +# The heap size of the jvm stared by hive shell script can be controlled via: + +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). + + +# Set HADOOP_HOME to point to a specific hadoop install directory +HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + +# Hive Configuration Directory can be controlled by: +export HIVE_CONF_DIR={{hive_config_dir}} + +# Folder containing extra libraries required for hive compilation/execution can be controlled by: +if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + if [ -f "${HIVE_AUX_JARS_PATH}" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} + elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar + fi +elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar +fi + +export METASTORE_PORT={{hive_metastore_port}} + +{% if sqla_db_used %} +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}" +export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}" +{% endif %} + </value> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d304151b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml index eaba2bc..e65c9ef 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml @@ -29,4 +29,16 @@ limitations under the License. </description> </property> + <property> + <name>datanucleus.rdbms.datastoreAdapterClassName</name> + <value>org.datanucleus.store.rdbms.adapter.SQLAnywhereAdapter</value> + <description>Datanucleus Class, This property used only when hive db is SQLA</description> + <depends-on> + <property> + <type>hive-site</type> + <name>javax.jdo.option.ConnectionDriverName</name> + </property> + </depends-on> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/d304151b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 9329032..9a6602a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -161,6 +161,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor): super(HDP23StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts) putHiveSiteProperty = self.putProperty(configurations, "hive-site", services) putHiveServerProperty = self.putProperty(configurations, "hiveserver2-site", services) + hive_site_properties = getSiteProperties(configurations, "hive-site") + putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site") servicesList = [service["StackServices"]["service_name"] for service in services["services"]] # hive_security_authorization == 'ranger' if str(configurations["hive-env"]["properties"]["hive_security_authorization"]).lower() == "ranger": @@ -178,6 +180,13 @@ class HDP23StackAdvisor(HDP22StackAdvisor): jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB" putHiveSiteProperty('hive.tez.java.opts', "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA " + jvmGCParams + " -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps") + # if hive using sqla db, then we should add DataNucleus property + sqla_db_used = 'javax.jdo.option.ConnectionDriverName' in hive_site_properties and \ + hive_site_properties['javax.jdo.option.ConnectionDriverName'] == 'sap.jdbc4.sqlanywhere.IDriver' + if sqla_db_used: + putHiveSiteProperty('datanucleus.rdbms.datastoreAdapterClassName','org.datanucleus.store.rdbms.adapter.SQLAnywhereAdapter') + else: + putHiveSitePropertyAttribute('datanucleus.rdbms.datastoreAdapterClassName', 'delete', 'true') def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts): super(HDP23StackAdvisor, self).recommendHDFSConfigurations(configurations, clusterData, services, hosts) @@ -202,7 +211,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor): parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators() childValidators = { "HDFS": {"hdfs-site": self.validateHDFSConfigurations}, - "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations}, + "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations, + "hive-site": self.validateHiveConfigurations}, "HBASE": {"hbase-site": self.validateHBASEConfigurations}, "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations} } @@ -227,6 +237,28 @@ class HDP23StackAdvisor(HDP22StackAdvisor): return self.toConfigurationValidationProblems(validationItems, "hdfs-site") + def validateHiveConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + super(HDP23StackAdvisor, self).validateHiveConfigurations(properties, recommendedDefaults, configurations, services, hosts) + hive_site = properties + validationItems = [] + sqla_db_used = "javax.jdo.option.ConnectionDriverName" in hive_site and \ + hive_site['javax.jdo.option.ConnectionDriverName'] == 'sap.jdbc4.sqlanywhere.IDriver' + prop_name = "datanucleus.rdbms.datastoreAdapterClassName" + prop_value = "org.datanucleus.store.rdbms.adapter.SQLAnywhereAdapter" + if sqla_db_used: + if not prop_name in hive_site: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Hive using SQLA db." \ + " {0} needs to be added with value {1}".format(prop_name,prop_value))}) + elif prop_name in hive_site and hive_site[prop_name] != "org.datanucleus.store.rdbms.adapter.SQLAnywhereAdapter": + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Hive using SQLA db." \ + " {0} needs to be set to {1}".format(prop_name,prop_value))}) + return self.toConfigurationValidationProblems(validationItems, "hive-site") + + def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts): super(HDP23StackAdvisor, self).validateHiveServer2Configurations(properties, recommendedDefaults, configurations, services, hosts) hive_server2 = properties http://git-wip-us.apache.org/repos/asf/ambari/blob/d304151b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 7f948bd..08c48d8 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -331,7 +331,8 @@ class TestHDP23StackAdvisor(TestCase): 'hive.server2.authentication.ldap.url': {'delete': 'true'}, 'hive.server2.tez.default.queues': { 'entries': [{'value': 'queue1', 'label': 'queue1 queue'}, {'value': 'queue2', 'label': 'queue2 queue'}] - } + }, + 'datanucleus.rdbms.datastoreAdapterClassName': {'delete': 'true'} } }, 'hiveserver2-site': {