http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py new file mode 100644 index 0000000..dd0031c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from resource_management import Script + +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.core.resources.accounts import User +from resource_management.core.resources.system import Directory, File, Execute +from resource_management.core.source import Template + + + +class Pxf(Script): + """ + Contains the interface definitions for methods like install, + start, stop, status, etc. for the PXF + """ + + def install(self, env): + self.install_packages(env) + self.configure(env) + + + def configure(self, env): + import params + env.set_params(params) + self.__setup_user_group() + self.__generate_config_files() + # pxf-service init exits safely when it is already initialized + self.__execute_service_command("init") + + + def start(self, env): + self.configure(env) + self.__grant_permissions() + self.__execute_service_command("restart") + + + def stop(self, env): + self.__execute_service_command("stop") + + + def status(self, env): + try: + self.__execute_service_command("status") + except Exception: + raise ComponentIsNotRunning() + + + def __execute_service_command(self, command): + import params + Execute("service {0} {1}".format(params.pxf_service_name, command), + timeout=params.default_exec_timeout, + logoutput=True) + + + def __setup_user_group(self): + """ + Creates PXF user with the required groups and bash as default shell + """ + import params + User(params.pxf_user, + groups=[params.hdfs_superuser_group, params.user_group, params.tomcat_group], + shell="/bin/bash") + + + def __generate_config_files(self): + """ + Generates pxf-env.sh file from jinja template and sets the classpath for HDP + """ + import params + import shutil + + hdp_stack = "HDP" + + # Create file pxf-env.sh from jinja template + File("{0}/pxf-env.sh".format(params.pxf_conf_dir), + content = Template("pxf-env.j2")) + + # Classpath is set for PHD by default. If stack is HDP, set classpath for HDP + if(params.stack_name == hdp_stack): + shutil.copy2("{0}/pxf-privatehdp.classpath".format(params.pxf_conf_dir), + "{0}/pxf-private.classpath".format(params.pxf_conf_dir)) + + XmlConfig("pxf-site.xml", + conf_dir=params.pxf_conf_dir, + configurations=params.config['configurations']['pxf-site'], + configuration_attributes=params.config['configuration_attributes']['pxf-site']) + + + def __grant_permissions(self): + """ + Grants permission to pxf:pxf for PXF instance directory + """ + import params + Directory(params.pxf_instance_dir, + owner=params.pxf_user, + group=params.pxf_group, + recursive=True) + + +if __name__ == "__main__": + Pxf().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/templates/pxf-env.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/templates/pxf-env.j2 b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/templates/pxf-env.j2 new file mode 100644 index 0000000..03f2420 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/templates/pxf-env.j2 @@ -0,0 +1,34 @@ +#!/bin/sh + +#Licensed to the Apache Software Foundation (ASF) under one +#or more contributor license agreements. See the NOTICE file +#distributed with this work for additional information +#regarding copyright ownership. The ASF licenses this file +#to you under the Apache License, Version 2.0 (the +#"License"); you may not use this file except in compliance +#with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. + + +# THIS FILE SHOULD MATCH WITH https://github.com/apache/incubator-hawq/blob/master/pxf/pxf-service/src/scripts/pxf-env.sh + +# Path to HDFS native libraries +export LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:${LD_LIBRARY_PATH} + +# Path to JAVA +export JAVA_HOME={{java_home}} + +# Path to Log directory +export PXF_LOGDIR=/var/log/pxf +export CATALINA_OUT=${PXF_LOGDIR}/catalina.out + +# Path to Run directory +export PXF_RUNDIR=/var/run/pxf + http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json index 0afe4c2..bfe286b 100755 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json @@ -8,6 +8,9 @@ "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"], "PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"], "ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"], - "SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"] + "SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"], + "HAWQMASTER-START" : ["NAMENODE-START","DATANODE-START","HAWQSTANDBY-START"], + "HAWQSEGMENT-START" : ["HAWQMASTER-START","HAWQSTANDBY-START"], + "HAWQ_SERVICE_CHECK-SERVICE_CHECK" : ["HAWQMASTER-START"] } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index d9c5ccc..36432c6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -22,6 +22,76 @@ import socket class HDP23StackAdvisor(HDP22StackAdvisor): + def createComponentLayoutRecommendations(self, services, hosts): + parentComponentLayoutRecommendations = super(HDP23StackAdvisor, self).createComponentLayoutRecommendations(services, hosts) + + # remove HAWQSTANDBY on a single node + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + if len(hostsList) == 1: + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if "HAWQ" in servicesList: + components = parentComponentLayoutRecommendations["blueprint"]["host_groups"][0]["components"] + components = [ component for component in components if component["name"] != 'HAWQSTANDBY' ] + parentComponentLayoutRecommendations["blueprint"]["host_groups"][0]["components"] = components + + return parentComponentLayoutRecommendations + + def getComponentLayoutValidations(self, services, hosts): + parentItems = super(HDP23StackAdvisor, self).getComponentLayoutValidations(services, hosts) + + if not "HAWQ" in [service["StackServices"]["service_name"] for service in services["services"]]: + return parentItems + + childItems = [] + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + hostsCount = len(hostsList) + + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"] + hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"] + + # single node case is not analyzed because HAWQ Standby Master will not be present in single node topology due to logic in createComponentLayoutRecommendations() + if len(hawqMasterHosts) > 0 and len(hawqStandbyHosts) > 0: + commonHosts = [host for host in hawqMasterHosts[0] if host in hawqStandbyHosts[0]] + for host in commonHosts: + message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host." + childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": host } ) + + if len(hawqMasterHosts) > 0 and hostsCount > 1: + ambariServerHosts = [host for host in hawqMasterHosts[0] if self.isLocalHost(host)] + for host in ambariServerHosts: + message = "HAWQ Master and Ambari Server should not be deployed on the same host. " \ + "If you leave them collocated, make sure to set HAWQ Master Port property " \ + "to a value different from the port number used by Ambari Server database." + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": host } ) + + if len(hawqStandbyHosts) > 0 and hostsCount > 1: + ambariServerHosts = [host for host in hawqStandbyHosts[0] if self.isLocalHost(host)] + for host in ambariServerHosts: + message = "HAWQ Standby Master and Ambari Server should not be deployed on the same host. " \ + "If you leave them collocated, make sure to set HAWQ Master Port property " \ + "to a value different from the port number used by Ambari Server database." + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": host } ) + + parentItems.extend(childItems) + return parentItems + + def getNotPreferableOnServerComponents(self): + parentComponents = super(HDP23StackAdvisor, self).getNotPreferableOnServerComponents() + parentComponents.extend(['HAWQMASTER', 'HAWQSTANDBY']) + return parentComponents + + def getComponentLayoutSchemes(self): + parentSchemes = super(HDP23StackAdvisor, self).getComponentLayoutSchemes() + # key is max number of cluster hosts + 1, value is index in host list where to put the component + childSchemes = { + 'HAWQMASTER' : {6: 2, 31: 1, "else": 5}, + 'HAWQSTANDBY': {6: 1, 31: 2, "else": 3} + } + parentSchemes.update(childSchemes) + return parentSchemes + def getServiceConfigurationRecommenderDict(self): parentRecommendConfDict = super(HDP23StackAdvisor, self).getServiceConfigurationRecommenderDict() childRecommendConfDict = { http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json b/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json new file mode 100644 index 0000000..5d6a5b6 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.3/common/hosts-1-host.json @@ -0,0 +1,93 @@ +{ + "href" : "/api/v1/hosts?fields=Hosts/*&Hosts/host_name.in(c6401.ambari.apache.org)", + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "desired_configs" : null, + "disk_info" : [ + { + "available" : "481199976", + "device" : "/dev/mapper/VolGroup-lv_root", + "used" : "5713880", + "percent" : "2%", + "size" : "512971376", + "type" : "ext4", + "mountpoint" : "/" + }, + { + "available" : "1478456", + "device" : "tmpfs", + "used" : "0", + "percent" : "0%", + "size" : "1478456", + "type" : "tmpfs", + "mountpoint" : "/dev/shm" + }, + { + "available" : "438284", + "device" : "/dev/sda1", + "used" : "31960", + "percent" : "7%", + "size" : "495844", + "type" : "ext4", + "mountpoint" : "/boot" + }, + { + "available" : "191037148", + "device" : "vagrant", + "used" : "296051072", + "percent" : "61%", + "size" : "487088220", + "type" : "vboxsf", + "mountpoint" : "/vagrant" + } + ], + "host_health_report" : "", + "host_name" : "c6401.ambari.apache.org", + "host_state" : "HEALTHY", + "host_status" : "HEALTHY", + "ip" : "192.168.64.101", + "last_agent_env" : { + "stackFoldersAndFiles" : [ ], + "alternatives" : [ ], + "existingUsers" : [ ], + "existingRepos" : [ ], + "installedPackages" : [ ], + "hostHealth" : { + "activeJavaProcs" : [ ], + "agentTimeStampAtReporting" : 1445288922364, + "serverTimeStampAtReporting" : 1445288922425, + "liveServices" : [ + { + "desc" : "", + "name" : "ntpd", + "status" : "Healthy" + } + ] + }, + "umask" : 18, + "transparentHugePage" : "always", + "firewallRunning" : false, + "firewallName" : "iptables", + "reverseLookup" : true + }, + "last_heartbeat_time" : 1445288922425, + "last_registration_time" : 1445288888619, + "os_arch" : "x86_64", + "os_family" : "redhat6", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "recovery_report" : { + "summary" : "DISABLED", + "component_reports" : [ ] + }, + "recovery_summary" : "DISABLED", + "total_mem" : 2956916 + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json b/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json new file mode 100644 index 0000000..3c0511e --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.3/common/hosts-3-hosts.json @@ -0,0 +1,269 @@ +{ + "href" : "/api/v1/hosts?fields=Hosts/*&Hosts/host_name.in(c6401.ambari.apache.org,c6402.ambari.apache.org,c6403.ambari.apache.org)", + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "desired_configs" : null, + "disk_info" : [ + { + "available" : "481199416", + "device" : "/dev/mapper/VolGroup-lv_root", + "used" : "5714440", + "percent" : "2%", + "size" : "512971376", + "type" : "ext4", + "mountpoint" : "/" + }, + { + "available" : "1478456", + "device" : "tmpfs", + "used" : "0", + "percent" : "0%", + "size" : "1478456", + "type" : "tmpfs", + "mountpoint" : "/dev/shm" + }, + { + "available" : "438284", + "device" : "/dev/sda1", + "used" : "31960", + "percent" : "7%", + "size" : "495844", + "type" : "ext4", + "mountpoint" : "/boot" + }, + { + "available" : "191426876", + "device" : "vagrant", + "used" : "295661344", + "percent" : "61%", + "size" : "487088220", + "type" : "vboxsf", + "mountpoint" : "/vagrant" + } + ], + "host_health_report" : "", + "host_name" : "c6401.ambari.apache.org", + "host_state" : "HEALTHY", + "host_status" : "HEALTHY", + "ip" : "192.168.64.101", + "last_agent_env" : { + "stackFoldersAndFiles" : [ ], + "alternatives" : [ ], + "existingUsers" : [ ], + "existingRepos" : [ ], + "installedPackages" : [ ], + "hostHealth" : { + "activeJavaProcs" : [ ], + "agentTimeStampAtReporting" : 1445290054919, + "serverTimeStampAtReporting" : 1445290054980, + "liveServices" : [ + { + "desc" : "", + "name" : "ntpd", + "status" : "Healthy" + } + ] + }, + "umask" : 18, + "transparentHugePage" : "always", + "firewallRunning" : false, + "firewallName" : "iptables", + "reverseLookup" : true + }, + "last_heartbeat_time" : 1445290077648, + "last_registration_time" : 1445289898391, + "os_arch" : "x86_64", + "os_family" : "redhat6", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "recovery_report" : { + "summary" : "DISABLED", + "component_reports" : [ ] + }, + "recovery_summary" : "DISABLED", + "total_mem" : 2956916 + } + }, + { + "href" : "/api/v1/hosts/c6402.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "desired_configs" : null, + "disk_info" : [ + { + "available" : "482615388", + "device" : "/dev/mapper/VolGroup-lv_root", + "used" : "4298468", + "percent" : "1%", + "size" : "512971376", + "type" : "ext4", + "mountpoint" : "/" + }, + { + "available" : "1478456", + "device" : "tmpfs", + "used" : "0", + "percent" : "0%", + "size" : "1478456", + "type" : "tmpfs", + "mountpoint" : "/dev/shm" + }, + { + "available" : "438284", + "device" : "/dev/sda1", + "used" : "31960", + "percent" : "7%", + "size" : "495844", + "type" : "ext4", + "mountpoint" : "/boot" + }, + { + "available" : "191430912", + "device" : "vagrant", + "used" : "295657308", + "percent" : "61%", + "size" : "487088220", + "type" : "vboxsf", + "mountpoint" : "/vagrant" + } + ], + "host_health_report" : "", + "host_name" : "c6402.ambari.apache.org", + "host_state" : "HEALTHY", + "host_status" : "HEALTHY", + "ip" : "192.168.64.102", + "last_agent_env" : { + "stackFoldersAndFiles" : [ ], + "alternatives" : [ ], + "existingUsers" : [ ], + "existingRepos" : [ ], + "installedPackages" : [ ], + "hostHealth" : { + "activeJavaProcs" : [ ], + "agentTimeStampAtReporting" : 1445290076132, + "serverTimeStampAtReporting" : 1445290076343, + "liveServices" : [ + { + "desc" : "", + "name" : "ntpd", + "status" : "Healthy" + } + ] + }, + "umask" : 18, + "transparentHugePage" : "always", + "firewallRunning" : false, + "firewallName" : "iptables", + "reverseLookup" : true + }, + "last_heartbeat_time" : 1445290086404, + "last_registration_time" : 1445290032489, + "os_arch" : "x86_64", + "os_family" : "redhat6", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6402.ambari.apache.org", + "rack_info" : "/default-rack", + "recovery_report" : { + "summary" : "DISABLED", + "component_reports" : [ ] + }, + "recovery_summary" : "DISABLED", + "total_mem" : 2956916 + } + }, + { + "href" : "/api/v1/hosts/c6403.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "desired_configs" : null, + "disk_info" : [ + { + "available" : "482615388", + "device" : "/dev/mapper/VolGroup-lv_root", + "used" : "4298468", + "percent" : "1%", + "size" : "512971376", + "type" : "ext4", + "mountpoint" : "/" + }, + { + "available" : "1478456", + "device" : "tmpfs", + "used" : "0", + "percent" : "0%", + "size" : "1478456", + "type" : "tmpfs", + "mountpoint" : "/dev/shm" + }, + { + "available" : "438284", + "device" : "/dev/sda1", + "used" : "31960", + "percent" : "7%", + "size" : "495844", + "type" : "ext4", + "mountpoint" : "/boot" + }, + { + "available" : "191430912", + "device" : "vagrant", + "used" : "295657308", + "percent" : "61%", + "size" : "487088220", + "type" : "vboxsf", + "mountpoint" : "/vagrant" + } + ], + "host_health_report" : "", + "host_name" : "c6403.ambari.apache.org", + "host_state" : "HEALTHY", + "host_status" : "HEALTHY", + "ip" : "192.168.64.103", + "last_agent_env" : { + "stackFoldersAndFiles" : [ ], + "alternatives" : [ ], + "existingUsers" : [ ], + "existingRepos" : [ ], + "installedPackages" : [ ], + "hostHealth" : { + "activeJavaProcs" : [ ], + "agentTimeStampAtReporting" : 1445290073223, + "serverTimeStampAtReporting" : 1445290073435, + "liveServices" : [ + { + "desc" : "", + "name" : "ntpd", + "status" : "Healthy" + } + ] + }, + "umask" : 18, + "transparentHugePage" : "always", + "firewallRunning" : false, + "firewallName" : "iptables", + "reverseLookup" : true + }, + "last_heartbeat_time" : 1445290083492, + "last_registration_time" : 1445290030522, + "os_arch" : "x86_64", + "os_family" : "redhat6", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6403.ambari.apache.org", + "rack_info" : "/default-rack", + "recovery_report" : { + "summary" : "DISABLED", + "component_reports" : [ ] + }, + "recovery_summary" : "DISABLED", + "total_mem" : 2956916 + } + } + ] +} \ No newline at end of file