Repository: ambari Updated Branches: refs/heads/branch-windows-dev 9213dccaf -> fd51370a2
AMBARI-7848. Add stack advisor to HDPWIN 2.1 stack. (Jayush Luniya via yusaku) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fd51370a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fd51370a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fd51370a Branch: refs/heads/branch-windows-dev Commit: fd51370a26dee4ae5649d5a031c2993dde652e11 Parents: 9213dcc Author: Yusaku Sako <yus...@hortonworks.com> Authored: Mon Oct 20 10:27:39 2014 -0700 Committer: Yusaku Sako <yus...@hortonworks.com> Committed: Mon Oct 20 10:27:39 2014 -0700 ---------------------------------------------------------------------- .../main/python/ambari_agent/HostInfo_win.py | 4 + .../apscheduler/jobstores/__init__.py | 18 + ambari-server/conf/windows/ambari.properties | 4 + .../src/main/assemblies/server-windows.xml | 1 + .../stackadvisor/StackAdvisorRunner.java | 9 +- .../src/main/python/ambari-server-windows.py | 3 +- .../stacks/HDPWIN/2.1/services/stack_advisor.py | 414 +++++++++++++++++++ 7 files changed, 449 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fd51370a/ambari-agent/src/main/python/ambari_agent/HostInfo_win.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo_win.py b/ambari-agent/src/main/python/ambari_agent/HostInfo_win.py index 2140426..fbaf743 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostInfo_win.py +++ b/ambari-agent/src/main/python/ambari_agent/HostInfo_win.py @@ -119,6 +119,10 @@ class HostInfo: result['status'] = "Available" results.append(result) + def createAlerts(self, alerts): + #TODO AMBARI-7849 Implement createAlerts for Windows + return alerts + def javaProcs(self, list): try: runner = shellRunner() http://git-wip-us.apache.org/repos/asf/ambari/blob/fd51370a/ambari-agent/src/main/python/ambari_agent/apscheduler/jobstores/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/apscheduler/jobstores/__init__.py b/ambari-agent/src/main/python/ambari_agent/apscheduler/jobstores/__init__.py new file mode 100644 index 0000000..0a0e1ca --- /dev/null +++ b/ambari-agent/src/main/python/ambari_agent/apscheduler/jobstores/__init__.py @@ -0,0 +1,18 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/fd51370a/ambari-server/conf/windows/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties index 9116ae9..0cd93bb 100644 --- a/ambari-server/conf/windows/ambari.properties +++ b/ambari-server/conf/windows/ambari.properties @@ -48,6 +48,10 @@ server.execution.scheduler.maxThreads=5 server.execution.scheduler.maxDbConnections=5 server.execution.scheduler.misfire.toleration.minutes=480 +recommendations.dir=\\var\\run\\ambari-server\\stack-recommendations +stackadvisor.script=resources\\scripts\\stack_advisor.py +server.tmp.dir=\\var\\run\\ambari-server\\tmp + # Default timeout in seconds before task is killed agent.task.timeout=600 http://git-wip-us.apache.org/repos/asf/ambari/blob/fd51370a/ambari-server/src/main/assemblies/server-windows.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/assemblies/server-windows.xml b/ambari-server/src/main/assemblies/server-windows.xml index 8ddf609..f04f1a5 100644 --- a/ambari-server/src/main/assemblies/server-windows.xml +++ b/ambari-server/src/main/assemblies/server-windows.xml @@ -151,6 +151,7 @@ <include>Ambari-DDL-SQLServer-*.sql</include> <include>custom_action_definitions/**</include> <include>custom_actions/**</include> + <include>stacks/stack_advisor.py</include> <include>scripts/**</include> <include>stacks/HDPWIN/**</include> <include>upgrade/**</include> http://git-wip-us.apache.org/repos/asf/ambari/blob/fd51370a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRunner.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRunner.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRunner.java index 97f688b..191aeed 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRunner.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRunner.java @@ -126,8 +126,13 @@ public class StackAdvisorRunner { // includes the original command plus the arguments for it List<String> builderParameters = new ArrayList<String>(); - builderParameters.add("sh"); - builderParameters.add("-c"); + if (System.getProperty("os.name").contains("Windows")) { + builderParameters.add("cmd"); + builderParameters.add("/c"); + } else { + builderParameters.add("sh"); + builderParameters.add("-c"); + } // for the 3rd argument, build a single parameter since we use -c // ProcessBuilder doesn't support output redirection until JDK 1.7 http://git-wip-us.apache.org/repos/asf/ambari/blob/fd51370a/ambari-server/src/main/python/ambari-server-windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server-windows.py b/ambari-server/src/main/python/ambari-server-windows.py index 0a7ed73..5f05fdd 100644 --- a/ambari-server/src/main/python/ambari-server-windows.py +++ b/ambari-server/src/main/python/ambari-server-windows.py @@ -596,5 +596,4 @@ if __name__ == "__main__": main() except (KeyboardInterrupt, EOFError): print("\nAborting ... Keyboard Interrupt.") - sys.exit(1) -ìï¿ë¾¿ï¾¿ë®»ï¾»ë·ï¾·ì§ï¿ïµï¿µéï¾ \ No newline at end of file + sys.exit(1) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/fd51370a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py new file mode 100644 index 0000000..cf35e47 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py @@ -0,0 +1,414 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import re +import sys +from math import ceil + +from stack_advisor import DefaultStackAdvisor + +class HDPWIN21StackAdvisor(DefaultStackAdvisor): + + def getComponentLayoutValidations(self, services, hosts): + """Returns array of Validation objects about issues with hostnames components assigned to""" + items = [] + + # Validating NAMENODE and SECONDARY_NAMENODE are on different hosts if possible + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + hostsCount = len(hostsList) + + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + nameNodeHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "NAMENODE"] + secondaryNameNodeHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "SECONDARY_NAMENODE"] + + # Validating cardinality + for component in componentsList: + if component["StackServiceComponents"]["cardinality"] is not None: + componentName = component["StackServiceComponents"]["component_name"] + componentDisplayName = component["StackServiceComponents"]["display_name"] + componentHostsCount = 0 + if component["StackServiceComponents"]["hostnames"] is not None: + componentHostsCount = len(component["StackServiceComponents"]["hostnames"]) + cardinality = str(component["StackServiceComponents"]["cardinality"]) + # cardinality types: null, 1+, 1-2, 1, ALL + message = None + if "+" in cardinality: + hostsMin = int(cardinality[:-1]) + if componentHostsCount < hostsMin: + message = "At least {0} {1} components should be installed in cluster.".format(hostsMin, componentDisplayName) + elif "-" in cardinality: + nums = cardinality.split("-") + hostsMin = int(nums[0]) + hostsMax = int(nums[1]) + if componentHostsCount > hostsMax or componentHostsCount < hostsMin: + message = "Between {0} and {1} {2} components should be installed in cluster.".format(hostsMin, hostsMax, componentDisplayName) + elif "ALL" == cardinality: + if componentHostsCount != hostsCount: + message = "{0} component should be installed on all hosts in cluster.".format(componentDisplayName) + else: + if componentHostsCount != int(cardinality): + message = "Exactly {0} {1} components should be installed in cluster.".format(int(cardinality), componentDisplayName) + + if message is not None: + items.append({"type": 'host-component', "level": 'ERROR', "message": message, "component-name": componentName}) + + # Validating host-usage + usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)] + usedHostsList = [item for sublist in usedHostsListList for item in sublist] + nonUsedHostsList = [item for item in hostsList if item not in usedHostsList] + for host in nonUsedHostsList: + items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not used', "host": str(host) } ) + + return items + + def getServiceConfigurationRecommenderDict(self): + return { + "YARN": self.recommendYARNConfigurations, + "MAPREDUCE2": self.recommendMapReduce2Configurations, + "OOZIE": self.recommendOozieConfigurations, + "HIVE": self.recommendHiveConfigurations, + "TEZ": self.recommendTezConfigurations + } + + def putProperty(self, config, configType): + config[configType] = {"properties": {}} + def appendProperty(key, value): + config[configType]["properties"][key] = str(value) + return appendProperty + + def recommendYARNConfigurations(self, configurations, clusterData): + putYarnProperty = self.putProperty(configurations, "yarn-site") + putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) + putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) + putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) + + def recommendMapReduce2Configurations(self, configurations, clusterData): + putMapredProperty = self.putProperty(configurations, "mapred-site") + putMapredProperty('yarn.app.mapreduce.am.resource.mb', int(clusterData['amMemory'])) + putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(round(0.8 * clusterData['amMemory']))) + "m") + putMapredProperty('mapreduce.map.memory.mb', clusterData['mapMemory']) + putMapredProperty('mapreduce.reduce.memory.mb', int(clusterData['reduceMemory'])) + putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m") + putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m") + putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024)) + + def recommendOozieConfigurations(self, configurations, clusterData): + if "FALCON_SERVER" in clusterData["components"]: + putMapredProperty = self.putProperty(configurations, "oozie-site") + putMapredProperty("oozie.services.ext", + "org.apache.oozie.service.JMSAccessorService," + + "org.apache.oozie.service.PartitionDependencyManagerService," + + "org.apache.oozie.service.HCatAccessorService") + + def recommendHiveConfigurations(self, configurations, clusterData): + containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory']) + containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize) + putHiveProperty = self.putProperty(configurations, "hive-site") + putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(round(containerSize / 3)) * 1048576) + putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(round(0.8 * containerSize))) + + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC") + putHiveProperty('hive.tez.container.size', containerSize) + + def recommendTezConfigurations(self, configurations, clusterData): + putTezProperty = self.putProperty(configurations, "tez-site") + putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory'])) + putTezProperty("tez.am.java.opts", + "-server -Xmx" + str(int(0.8 * clusterData["amMemory"])) + + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC") + + def getConfigurationClusterSummary(self, servicesList, hosts, components): + + hBaseInstalled = False + if 'HBASE' in servicesList: + hBaseInstalled = True + + cluster = { + "cpu": 0, + "disk": 0, + "ram": 0, + "hBaseInstalled": hBaseInstalled, + "components": components + } + + if len(hosts["items"]) > 0: + host = hosts["items"][0]["Hosts"] + cluster["cpu"] = host["cpu_count"] + cluster["disk"] = len(host["disk_info"]) + cluster["ram"] = int(host["total_mem"] / (1024 * 1024)) + + ramRecommendations = [ + {"os":1, "hbase":1}, + {"os":2, "hbase":1}, + {"os":2, "hbase":2}, + {"os":4, "hbase":4}, + {"os":6, "hbase":8}, + {"os":8, "hbase":8}, + {"os":8, "hbase":8}, + {"os":12, "hbase":16}, + {"os":24, "hbase":24}, + {"os":32, "hbase":32}, + {"os":64, "hbase":64} + ] + index = { + cluster["ram"] <= 4: 0, + 4 < cluster["ram"] <= 8: 1, + 8 < cluster["ram"] <= 16: 2, + 16 < cluster["ram"] <= 24: 3, + 24 < cluster["ram"] <= 48: 4, + 48 < cluster["ram"] <= 64: 5, + 64 < cluster["ram"] <= 72: 6, + 72 < cluster["ram"] <= 96: 7, + 96 < cluster["ram"] <= 128: 8, + 128 < cluster["ram"] <= 256: 9, + 256 < cluster["ram"]: 10 + }[1] + cluster["reservedRam"] = ramRecommendations[index]["os"] + cluster["hbaseRam"] = ramRecommendations[index]["hbase"] + + cluster["minContainerSize"] = { + cluster["ram"] <= 4: 256, + 4 < cluster["ram"] <= 8: 512, + 8 < cluster["ram"] <= 24: 1024, + 24 < cluster["ram"]: 2048 + }[1] + + totalAvailableRam = cluster["ram"] - cluster["reservedRam"] + if cluster["hBaseInstalled"]: + totalAvailableRam -= cluster["hbaseRam"] + cluster["totalAvailableRam"] = max(2048, totalAvailableRam * 1024) + '''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))''' + cluster["containers"] = round(max(3, + min(2 * cluster["cpu"], + min(ceil(1.8 * cluster["disk"]), + cluster["totalAvailableRam"] / cluster["minContainerSize"])))) + + '''ramPerContainers = max(2GB, RAM - reservedRam - hBaseRam) / containers''' + cluster["ramPerContainer"] = abs(cluster["totalAvailableRam"] / cluster["containers"]) + '''If greater than 1GB, value will be in multiples of 512.''' + if cluster["ramPerContainer"] > 1024: + cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / 512) * 512 + + cluster["mapMemory"] = int(cluster["ramPerContainer"]) + cluster["reduceMemory"] = cluster["ramPerContainer"] + cluster["amMemory"] = max(cluster["mapMemory"], cluster["reduceMemory"]) + + return cluster + + def getConfigurationsValidationItems(self, services, hosts): + """Returns array of Validation objects about issues with configuration values provided in services""" + items = [] + + recommendations = self.recommendConfigurations(services, hosts) + recommendedDefaults = recommendations["recommendations"]["blueprint"]["configurations"] + + configurations = services["configurations"] + for service in services["services"]: + serviceName = service["StackServices"]["service_name"] + validator = self.validateServiceConfigurations(serviceName) + if validator is not None: + siteName = validator[0] + method = validator[1] + if siteName in recommendedDefaults: + siteProperties = getSiteProperties(configurations, siteName) + if siteProperties is not None: + resultItems = method(siteProperties, recommendedDefaults[siteName]["properties"], configurations) + items.extend(resultItems) + return items + + def getServiceConfigurationValidators(self): + return { + "MAPREDUCE2": ["mapred-site", self.validateMapReduce2Configurations], + "YARN": ["yarn-site", self.validateYARNConfigurations] + "HIVE": ["hive-site", self.validateHiveConfigurations], + "TEZ": ["tez-site", self.validateTezConfigurations] + } + + def validateServiceConfigurations(self, serviceName): + return self.getServiceConfigurationValidators().get(serviceName, None) + + def toConfigurationValidationProblems(self, validationProblems, siteName): + result = [] + for validationProblem in validationProblems: + validationItem = validationProblem.get("item", None) + if validationItem is not None: + problem = {"type": 'configuration', "level": validationItem["level"], "message": validationItem["message"], + "config-type": siteName, "config-name": validationProblem["config-name"] } + result.append(problem) + return result + + def getWarnItem(self, message): + return {"level": "WARN", "message": message} + + def getErrorItem(self, message): + return {"level": "ERROR", "message": message} + + def validatorLessThenDefaultValue(self, properties, recommendedDefaults, propertyName): + if not propertyName in properties: + return self.getErrorItem("Value should be set") + value = to_number(properties[propertyName]) + if value is None: + return self.getErrorItem("Value should be integer") + defaultValue = to_number(recommendedDefaults[propertyName]) + if defaultValue is None: + return None + if value < defaultValue: + return self.getWarnItem("Value is less than the recommended default of {0}".format(defaultValue)) + return None + + def validateXmxValue(self, properties, recommendedDefaults, propertyName): + if not propertyName in properties: + return self.getErrorItem("Value should be set") + value = properties[propertyName] + defaultValue = recommendedDefaults[propertyName] + if defaultValue is None: + return self.getErrorItem("Config's default value can't be null or undefined") + if not checkXmxValueFormat(value): + return self.getErrorItem('Invalid value format') + valueInt = formatXmxSizeToBytes(getXmxSize(value)) + defaultValueXmx = getXmxSize(defaultValue) + defaultValueInt = formatXmxSizeToBytes(defaultValueXmx) + if valueInt < defaultValueInt: + return self.getWarnItem("Value is less than the recommended default of -Xmx" + defaultValueXmx) + return None + + def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations): + validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')}, + {"config-name": 'mapreduce.reduce.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')}, + {"config-name": 'mapreduce.task.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')}, + {"config-name": 'mapreduce.map.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')}, + {"config-name": 'mapreduce.reduce.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')}, + {"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')}, + {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ] + return self.toConfigurationValidationProblems(validationItems, "mapred-site") + + def validateYARNConfigurations(self, properties, recommendedDefaults, configurations): + validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')}, + {"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')}, + {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ] + return self.toConfigurationValidationProblems(validationItems, "yarn-site") + + def validateHiveConfigurations(self, properties, recommendedDefaults, configurations): + validationItems = [ {"config-name": 'hive.tez.container.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')}, + {"config-name": 'hive.tez.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')}, + {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ] + return self.toConfigurationValidationProblems(validationItems, "hive-site") + + def validateTezConfigurations(self, properties, recommendedDefaults, configurations): + validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')}, + {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ] + return self.toConfigurationValidationProblems(validationItems, "tez-site") + + def getMastersWithMultipleInstances(self): + return ['ZOOKEEPER_SERVER', 'HBASE_MASTER'] + + def getNotValuableComponents(self): + return ['JOURNALNODE', 'ZKFC', 'APP_TIMELINE_SERVER'] + + def getNotPreferableOnServerComponents(self): + return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS'] + + def getCardinalitiesDict(self): + return { + 'ZOOKEEPER_SERVER': {"min": 3}, + 'HBASE_MASTER': {"min": 1}, + } + + def getComponentLayoutSchemes(self): + return { + 'NAMENODE': {"else": 0}, + 'SECONDARY_NAMENODE': {"else": 1}, + 'HBASE_MASTER': {6: 0, 31: 2, "else": 3}, + + 'HISTORYSERVER': {31: 1, "else": 2}, + 'RESOURCEMANAGER': {31: 1, "else": 2}, + + 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3}, + + 'HIVE_SERVER': {6: 1, 31: 2, "else": 4}, + 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4}, + 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4}, + 'APP_TIMELINE_SERVER': {31: 1, "else": 2}, + 'FALCON_SERVER': {6: 1, 31: 2, "else": 3} + } + +# Validation helper methods +def getSiteProperties(configurations, siteName): + siteConfig = configurations.get(siteName) + if siteConfig is None: + return None + return siteConfig.get("properties") + +def to_number(s): + try: + return int(re.sub("\D", "", s)) + except ValueError: + return None + +def checkXmxValueFormat(value): + p = re.compile('-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)?') + matches = p.findall(value) + return len(matches) == 1 + +def getXmxSize(value): + p = re.compile("-Xmx(\d+)(.?)") + result = p.findall(value)[0] + if len(result) > 1: + # result[1] - is a space or size formatter (b|k|m|g etc) + return result[0] + result[1].lower() + return result[0] + +def formatXmxSizeToBytes(value): + value = value.lower() + if len(value) == 0: + return 0 + modifier = value[-1] + + if modifier == ' ' or modifier in "0123456789": + modifier = 'b' + m = { + modifier == 'b': 1, + modifier == 'k': 1024, + modifier == 'm': 1024 * 1024, + modifier == 'g': 1024 * 1024 * 1024, + modifier == 't': 1024 * 1024 * 1024 * 1024, + modifier == 'p': 1024 * 1024 * 1024 * 1024 * 1024 + }[1] + return to_number(value) * m + +def getPort(address): + """ + Extracts port from the address like 0.0.0.0:1019 + """ + if address is None: + return None + m = re.search(r'(?:http(?:s)?://)?([\w\d.]*):(\d{1,5})', address) + if m is not None: + return int(m.group(2)) + else: + return None + +def isSecurePort(port): + """ + Returns True if port is root-owned at *nix systems + """ + if port is not None: + return port < 1024 + else: + return False \ No newline at end of file
