Repository: ambari Updated Branches: refs/heads/trunk e8544ba49 -> 61442de4c
AMBARI-18597. Rename service to "Microsoft R Server" and component to "Microsoft R Node Client". (Balazs Bence Sari via stoader) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/61442de4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/61442de4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/61442de4 Branch: refs/heads/trunk Commit: 61442de4c6d8690595a5b331cda3890b9e26b5b9 Parents: e8544ba Author: Balazs Bence Sari <bs...@hortonworks.com> Authored: Mon Oct 17 10:59:39 2016 +0200 Committer: Toader, Sebastian <stoa...@hortonworks.com> Committed: Mon Oct 17 10:59:39 2016 +0200 ---------------------------------------------------------------------- .../scripts/shared_initialization.py | 2 +- .../MICROSOFT_R/8.0.5/kerberos.json | 15 ---- .../MICROSOFT_R/8.0.5/metainfo.xml | 53 ------------ .../package/files/microsoft_r_serviceCheck.r | 28 ------ .../8.0.5/package/scripts/microsoft_r.py | 38 --------- .../MICROSOFT_R/8.0.5/package/scripts/params.py | 31 ------- .../8.0.5/package/scripts/params_linux.py | 79 ----------------- .../8.0.5/package/scripts/params_windows.py | 26 ------ .../8.0.5/package/scripts/service_check.py | 89 -------------------- .../MICROSOFT_R/8.0.5/role_command_order.json | 6 -- .../MICROSOFT_R/8.0.5/service_advisor.py | 73 ---------------- .../MICROSOFT_R_SERVER/8.0.5/kerberos.json | 15 ++++ .../MICROSOFT_R_SERVER/8.0.5/metainfo.xml | 53 ++++++++++++ .../files/microsoft_r_server_serviceCheck.r | 28 ++++++ .../8.0.5/package/scripts/microsoft_r_server.py | 38 +++++++++ .../8.0.5/package/scripts/params.py | 31 +++++++ .../8.0.5/package/scripts/params_linux.py | 79 +++++++++++++++++ .../8.0.5/package/scripts/params_windows.py | 26 ++++++ .../8.0.5/package/scripts/service_check.py | 89 ++++++++++++++++++++ .../8.0.5/role_command_order.json | 6 ++ .../MICROSOFT_R_SERVER/8.0.5/service_advisor.py | 73 ++++++++++++++++ .../MICROSOFT_R/8.0.5/metainfo.xml | 47 ----------- .../MICROSOFT_R/8.0.5/repos/repoinfo.xml | 40 --------- .../MICROSOFT_R_SERVER/8.0.5/metainfo.xml | 47 +++++++++++ .../MICROSOFT_R_SERVER/8.0.5/repos/repoinfo.xml | 40 +++++++++ .../src/main/resources/mpack.json | 10 +-- 26 files changed, 531 insertions(+), 531 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py index 397c22d..5dce8e0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py @@ -177,7 +177,7 @@ def create_dirs(): def create_microsoft_r_dir(): import params - if 'MICROSOFT_R_SERVER_CLIENT' in params.component_list and params.default_fs: + if 'MICROSOFT_R_NODE_CLIENT' in params.component_list and params.default_fs: directory = '/user/RevoShare' try: params.HdfsResource(directory, http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/kerberos.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/kerberos.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/kerberos.json deleted file mode 100644 index 86fab2d..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/kerberos.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "services": [ - { - "name": "MICROSOFT_R", - "identities": [ - { - "name": "/HDFS/NAMENODE/hdfs" - }, - { - "name": "/smokeuser" - } - ] - } - ] -} http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/metainfo.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/metainfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/metainfo.xml deleted file mode 100644 index 6c5f062..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/metainfo.xml +++ /dev/null @@ -1,53 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<metainfo> - <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>MICROSOFT_R</name> - <displayName>Microsoft R</displayName> - <comment>Microsoft R Server</comment> - <version>8.0.5</version> - - <components> - <component> - <name>MICROSOFT_R_SERVER_CLIENT</name> - <displayName>Microsoft R Server Client</displayName> - <category>CLIENT</category> - <cardinality>1+</cardinality> - <commandScript> - <script>scripts/microsoft_r.py</script> - <scriptType>PYTHON</scriptType> - <timeout>1200</timeout> - </commandScript> - </component> - </components> - - <commandScript> - <script>scripts/service_check.py</script> - <scriptType>PYTHON</scriptType> - <timeout>300</timeout> - </commandScript> - - <requiredServices> - <service>YARN</service> - </requiredServices> - </service> - </services> -</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/files/microsoft_r_serviceCheck.r ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/files/microsoft_r_serviceCheck.r b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/files/microsoft_r_serviceCheck.r deleted file mode 100644 index e75c598..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/files/microsoft_r_serviceCheck.r +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License */ - -bigDataDirRoot <- "/tmp/share" -myHadoopCluster <- RxHadoopMR(consoleOutput=TRUE) -rxSetComputeContext(myHadoopCluster) -source <- system.file("SampleData/AirlineDemoSmall.csv", package="RevoScaleR") -inputDir <- file.path(bigDataDirRoot,"AirlineDemoSmall") -rxHadoopMakeDir(inputDir) -rxHadoopCopyFromLocal(source, inputDir) -hdfsFS <- RxHdfsFileSystem() -colInfo <- list(DayOfWeek = list(type = "factor", -levels = c("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"))) -airDS <- RxTextData(file = inputDir, missingValueString = "M", colInfo = colInfo, fileSystem = hdfsFS) -adsSummary <- rxSummary(~ArrDelay+CRSDepTime+DayOfWeek, data = airDS) -adsSummary \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/microsoft_r.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/microsoft_r.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/microsoft_r.py deleted file mode 100644 index f6eb33e..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/microsoft_r.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from resource_management.libraries.script import Script -from resource_management.core.logger import Logger -from resource_management.core.resources import Package - - -class MicrosoftR(Script): - - def install(self, env): - Logger.info('Installing R Server Client...') - tmp_dir = Script.tmp_dir - Logger.debug('Using temp dir: {0}'.format(tmp_dir)) - self.install_packages(env) - Logger.info('Installed R Server') - - def configure(self, env): - Logger.info('Configure R Server. Nothing to do.') - -if __name__ == "__main__": - MicrosoftR().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params.py deleted file mode 100644 index aa20797..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -from ambari_commons import OSCheck -from resource_management.libraries.functions.default import default - -if OSCheck.is_windows_family(): - from params_windows import * -else: - from params_linux import * - -host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) - -revo_share_hdfs_folder = "/user/RevoShare" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_linux.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_linux.py deleted file mode 100644 index 71f17b6..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_linux.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management.libraries.script.script import Script -from resource_management.libraries.resources.hdfs_resource import HdfsResource -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import StackFeature -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.version import format_stack_version -from resource_management.libraries.functions.format import format -from resource_management.libraries.functions.default import default -from resource_management.libraries.functions import get_kinit_path -from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources - -# server configurations -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() - -# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade -version = default("/commandParams/version", None) - -smokeuser = config['configurations']['cluster-env']['smokeuser'] -smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] -smoke_hdfs_user_mode = 0770 -user_group = config['configurations']['cluster-env']['user_group'] -security_enabled = config['configurations']['cluster-env']['security_enabled'] -smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) - -# not supporting 32 bit jdk. -java64_home = config['hostLevelParams']['java_home'] - -hadoop_bin_dir = stack_select.get_hadoop_dir("bin") -hdfs_user_principal = default('/configurations/hadoop-env/hdfs_principal_name', None) -hdfs_site = config['configurations']['hdfs-site'] -default_fs = config['configurations']['core-site']['fs.defaultFS'] -dfs_type = default("/commandParams/dfs_type", "") -hadoop_conf_dir = "/etc/hadoop/conf" - - -import functools -#create partial functions with common arguments for every HdfsResource call -#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code -HdfsResource = functools.partial( - HdfsResource, - user=hdfs_user, - hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", - security_enabled = security_enabled, - keytab = hdfs_user_keytab, - kinit_path_local = kinit_path_local, - hadoop_bin_dir = hadoop_bin_dir, - hadoop_conf_dir = hadoop_conf_dir, - principal_name = hdfs_user_principal, - hdfs_site = hdfs_site, - default_fs = default_fs, - immutable_paths = get_not_managed_resources(), - dfs_type = dfs_type -) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_windows.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_windows.py deleted file mode 100644 index 3eca213..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_windows.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -from resource_management.libraries.script.script import Script - -# server configurations -config = Script.get_config() - -stack_root = None http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/service_check.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/service_check.py deleted file mode 100644 index 5368e94..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/service_check.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import os - -from resource_management.core.exceptions import Fail -from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute, File -from resource_management.core.source import InlineTemplate, StaticFile -from resource_management.libraries.functions.format import format -from resource_management.libraries.script.script import Script -from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl - - -class MicrosoftRServiceCheck(Script): - pass - -@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) -class MicrosoftRServiceCheckLinux(MicrosoftRServiceCheck): - def service_check(self, env): - import params - env.set_params(params) - - try: - params.HdfsResource(params.revo_share_hdfs_folder, - type="directory", - action="create_on_execute", - owner=params.hdfs_user, - mode=0777) - params.HdfsResource(None, action="execute") - except Exception as exception: - Logger.warning("Could not check the existence of /user/RevoShare on HDFS, exception: {0}".format(str(exception))) - - if params.security_enabled: - kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};") - Execute(kinit_cmd, - user=params.smokeuser - ) - - output_file = format('{tmp_dir}/microsoft_r_serviceCheck.out') - - File( format("{tmp_dir}/microsoft_r_serviceCheck.r"), - content = StaticFile("microsoft_r_serviceCheck.r"), - mode = 0755 - ) - - Execute( format("Revo64 --no-save < {tmp_dir}/microsoft_r_serviceCheck.r | tee {output_file}"), - tries = 1, - try_sleep = 1, - path = format('/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'), - user = params.smokeuser, - logoutput = True - ) - - # Verify correct output - from resource_management.core import sudo - output_content = sudo.read_file(format('{output_file}')) - import re - values_list = re.findall(r"\s(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\s+(\d*)", output_content) - if 7 != len(values_list): - Logger.info("Unable to verify output of service check run") - raise Fail("Unable to verify output of service check run") - dayCountDictionary = {'Monday': '97975', 'Tuesday': '77725', 'Wednesday': '78875', 'Thursday': '81304', 'Friday': '82987', 'Saturday': '86159', 'Sunday': '94975'} - for (day, count) in values_list: - if count != dayCountDictionary[day]: - Logger.info("Service check produced incorrect output for {0}. Was expecting {1} but encountered {2}".format(day, dayCountDictionary[day], count)) - raise Fail("Service check produced incorrect output for {0}. Was expecting {1} but encountered {2}".format(day, dayCountDictionary[day], count)) - - -if __name__ == "__main__": - MicrosoftRServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/role_command_order.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/role_command_order.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/role_command_order.json deleted file mode 100755 index 6edbc2c..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/role_command_order.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "general_deps" : { - "_comment" : "dependencies for Microsoft R", - "MICROSOFT_R_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START"] - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/service_advisor.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/service_advisor.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/service_advisor.py deleted file mode 100644 index d48afec..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/service_advisor.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env ambari-python-wrap -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import os -import fnmatch -import imp -import socket -import sys -import traceback - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/') -PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') - -try: - with open(PARENT_FILE, 'rb') as fp: - service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) -except Exception as e: - traceback.print_exc() - print "Failed to load parent" - -class MICROSOFT_R805ServiceAdvisor(service_advisor.ServiceAdvisor): - - def colocateService(self, hostsComponentsMap, serviceComponents): - # colocate R_SERVER_CLIENT with NODEMANAGERs and YARN_CLIENTs - rClientComponent = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "MICROSOFT_R_SERVER_CLIENT"] - traceback.print_tb(None) - rClientComponent = rClientComponent[0] - if not self.isComponentHostsPopulated(rClientComponent): - for hostName in hostsComponentsMap.keys(): - hostComponents = hostsComponentsMap[hostName] - if ({"name": "NODEMANAGER"} in hostComponents or {"name": "YARN_CLIENT"} in hostComponents) \ - and {"name": "MICROSOFT_R_SERVER_CLIENT"} not in hostComponents: - hostsComponentsMap[hostName].append({ "name": "MICROSOFT_R_SERVER_CLIENT" }) - if ({"name": "NODEMANAGER"} not in hostComponents and {"name": "YARN_CLIENT"} not in hostComponents) \ - and {"name": "MICROSOFT_R_SERVER_CLIENT"} in hostComponents: - hostsComponentsMap[hostName].remove({"name": "MICROSOFT_R_SERVER_CLIENT"}) - - def getServiceComponentLayoutValidations(self, services, hosts): - componentsListList = [service["components"] for service in services["services"]] - componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] - hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] - hostsCount = len(hostsList) - - rClientHosts = self.getHosts(componentsList, "MICROSOFT_R_SERVER_CLIENT") - expectedrClientHosts = set(self.getHosts(componentsList, "NODEMANAGER")) | set(self.getHosts(componentsList, "YARN_CLIENT")) - - items = [] - - # Generate WARNING if any R_SERVER_CLIENT is not colocated with NODEMANAGER or YARN_CLIENT - mismatchHosts = sorted(expectedrClientHosts.symmetric_difference(set(rClientHosts))) - if len(mismatchHosts) > 0: - hostsString = ', '.join(mismatchHosts) - message = "Microsoft R Server Client must be installed on NodeManagers and YARN Clients. " \ - "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format(len(mismatchHosts), hostsString) - items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'MICROSOFT_R_SERVER_CLIENT' } ) - - return items http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/kerberos.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/kerberos.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/kerberos.json new file mode 100644 index 0000000..6e4aa4c --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/kerberos.json @@ -0,0 +1,15 @@ +{ + "services": [ + { + "name": "MICROSOFT_R_SERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "/smokeuser" + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/metainfo.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/metainfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/metainfo.xml new file mode 100644 index 0000000..9f29864 --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/metainfo.xml @@ -0,0 +1,53 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>MICROSOFT_R_SERVER</name> + <displayName>Microsoft R Server</displayName> + <comment>Microsoft R Server</comment> + <version>8.0.5</version> + + <components> + <component> + <name>MICROSOFT_R_NODE_CLIENT</name> + <displayName>Microsoft R Node Client</displayName> + <category>CLIENT</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/microsoft_r_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + </component> + </components> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>YARN</service> + </requiredServices> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/files/microsoft_r_server_serviceCheck.r ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/files/microsoft_r_server_serviceCheck.r b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/files/microsoft_r_server_serviceCheck.r new file mode 100644 index 0000000..e75c598 --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/files/microsoft_r_server_serviceCheck.r @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License */ + +bigDataDirRoot <- "/tmp/share" +myHadoopCluster <- RxHadoopMR(consoleOutput=TRUE) +rxSetComputeContext(myHadoopCluster) +source <- system.file("SampleData/AirlineDemoSmall.csv", package="RevoScaleR") +inputDir <- file.path(bigDataDirRoot,"AirlineDemoSmall") +rxHadoopMakeDir(inputDir) +rxHadoopCopyFromLocal(source, inputDir) +hdfsFS <- RxHdfsFileSystem() +colInfo <- list(DayOfWeek = list(type = "factor", +levels = c("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"))) +airDS <- RxTextData(file = inputDir, missingValueString = "M", colInfo = colInfo, fileSystem = hdfsFS) +adsSummary <- rxSummary(~ArrDelay+CRSDepTime+DayOfWeek, data = airDS) +adsSummary \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/microsoft_r_server.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/microsoft_r_server.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/microsoft_r_server.py new file mode 100644 index 0000000..223cbd8 --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/microsoft_r_server.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.libraries.script import Script +from resource_management.core.logger import Logger +from resource_management.core.resources import Package + + +class MicrosoftRServer(Script): + + def install(self, env): + Logger.info('Installing R Node Client...') + tmp_dir = Script.tmp_dir + Logger.debug('Using temp dir: {0}'.format(tmp_dir)) + self.install_packages(env) + Logger.info('Installed R Node Client') + + def configure(self, env): + Logger.info('Configure R Server. Nothing to do.') + +if __name__ == "__main__": + MicrosoftRServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params.py new file mode 100644 index 0000000..aa20797 --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params.py @@ -0,0 +1,31 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions.default import default + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) + +revo_share_hdfs_folder = "/user/RevoShare" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params_linux.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params_linux.py new file mode 100644 index 0000000..71f17b6 --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params_linux.py @@ -0,0 +1,79 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources + +# server configurations +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) + +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +smoke_hdfs_user_mode = 0770 +user_group = config['configurations']['cluster-env']['user_group'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + +# not supporting 32 bit jdk. +java64_home = config['hostLevelParams']['java_home'] + +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hdfs_user_principal = default('/configurations/hadoop-env/hdfs_principal_name', None) +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] +dfs_type = default("/commandParams/dfs_type", "") +hadoop_conf_dir = "/etc/hadoop/conf" + + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_user_principal, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources(), + dfs_type = dfs_type +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params_windows.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params_windows.py new file mode 100644 index 0000000..3eca213 --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/params_windows.py @@ -0,0 +1,26 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from resource_management.libraries.script.script import Script + +# server configurations +config = Script.get_config() + +stack_root = None http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/service_check.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/service_check.py new file mode 100644 index 0000000..57b6681 --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/package/scripts/service_check.py @@ -0,0 +1,89 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import os + +from resource_management.core.exceptions import Fail +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute, File +from resource_management.core.source import InlineTemplate, StaticFile +from resource_management.libraries.functions.format import format +from resource_management.libraries.script.script import Script +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + + +class MicrosoftRServerServiceCheck(Script): + pass + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class MicrosoftRServerServiceCheckLinux(MicrosoftRServerServiceCheck): + def service_check(self, env): + import params + env.set_params(params) + + try: + params.HdfsResource(params.revo_share_hdfs_folder, + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + mode=0777) + params.HdfsResource(None, action="execute") + except Exception as exception: + Logger.warning("Could not check the existence of /user/RevoShare on HDFS, exception: {0}".format(str(exception))) + + if params.security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};") + Execute(kinit_cmd, + user=params.smokeuser + ) + + output_file = format('{tmp_dir}/microsoft_r_server_serviceCheck.out') + + File( format("{tmp_dir}/microsoft_r_server_serviceCheck.r"), + content = StaticFile("microsoft_r_server_serviceCheck.r"), + mode = 0755 + ) + + Execute( format("Revo64 --no-save < {tmp_dir}/microsoft_r_server_serviceCheck.r | tee {output_file}"), + tries = 1, + try_sleep = 1, + path = format('/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'), + user = params.smokeuser, + logoutput = True + ) + + # Verify correct output + from resource_management.core import sudo + output_content = sudo.read_file(format('{output_file}')) + import re + values_list = re.findall(r"\s(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\s+(\d*)", output_content) + if 7 != len(values_list): + Logger.info("Unable to verify output of service check run") + raise Fail("Unable to verify output of service check run") + dayCountDictionary = {'Monday': '97975', 'Tuesday': '77725', 'Wednesday': '78875', 'Thursday': '81304', 'Friday': '82987', 'Saturday': '86159', 'Sunday': '94975'} + for (day, count) in values_list: + if count != dayCountDictionary[day]: + Logger.info("Service check produced incorrect output for {0}. Was expecting {1} but encountered {2}".format(day, dayCountDictionary[day], count)) + raise Fail("Service check produced incorrect output for {0}. Was expecting {1} but encountered {2}".format(day, dayCountDictionary[day], count)) + + +if __name__ == "__main__": + MicrosoftRServerServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/role_command_order.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/role_command_order.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/role_command_order.json new file mode 100755 index 0000000..c24ae31 --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/role_command_order.json @@ -0,0 +1,6 @@ +{ + "general_deps" : { + "_comment" : "dependencies for Microsoft R Server", + "MICROSOFT_R_SERVER_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START"] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/service_advisor.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/service_advisor.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/service_advisor.py new file mode 100644 index 0000000..0ecc310 --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R_SERVER/8.0.5/service_advisor.py @@ -0,0 +1,73 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +import fnmatch +import imp +import socket +import sys +import traceback + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print "Failed to load parent" + +class MICROSOFT_R_SERVER805ServiceAdvisor(service_advisor.ServiceAdvisor): + + def colocateService(self, hostsComponentsMap, serviceComponents): + # colocate R_NODE_CLIENT with NODEMANAGERs and YARN_CLIENTs + rClientComponent = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "MICROSOFT_R_NODE_CLIENT"] + traceback.print_tb(None) + rClientComponent = rClientComponent[0] + if not self.isComponentHostsPopulated(rClientComponent): + for hostName in hostsComponentsMap.keys(): + hostComponents = hostsComponentsMap[hostName] + if ({"name": "NODEMANAGER"} in hostComponents or {"name": "YARN_CLIENT"} in hostComponents) \ + and {"name": "MICROSOFT_R_NODE_CLIENT"} not in hostComponents: + hostsComponentsMap[hostName].append({ "name": "MICROSOFT_R_NODE_CLIENT" }) + if ({"name": "NODEMANAGER"} not in hostComponents and {"name": "YARN_CLIENT"} not in hostComponents) \ + and {"name": "MICROSOFT_R_NODE_CLIENT"} in hostComponents: + hostsComponentsMap[hostName].remove({"name": "MICROSOFT_R_NODE_CLIENT"}) + + def getServiceComponentLayoutValidations(self, services, hosts): + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + hostsCount = len(hostsList) + + rClientHosts = self.getHosts(componentsList, "MICROSOFT_R_NODE_CLIENT") + expectedrClientHosts = set(self.getHosts(componentsList, "NODEMANAGER")) | set(self.getHosts(componentsList, "YARN_CLIENT")) + + items = [] + + # Generate WARNING if any R_NODE_CLIENT is not colocated with NODEMANAGER or YARN_CLIENT + mismatchHosts = sorted(expectedrClientHosts.symmetric_difference(set(rClientHosts))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "Microsoft R Node Client must be installed on NodeManagers and YARN Clients. " \ + "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format(len(mismatchHosts), hostsString) + items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'MICROSOFT_R_NODE_CLIENT' } ) + + return items http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/metainfo.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/metainfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/metainfo.xml deleted file mode 100644 index 9470d22..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/metainfo.xml +++ /dev/null @@ -1,47 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<metainfo> - <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>MICROSOFT_R</name> - <extends>common-services/MICROSOFT_R/8.0.5</extends> - <version>8.0.5.0</version> - <osSpecifics> - <osSpecific> - <osFamily>redhat6,redhat7,suse11</osFamily> - <packages> - <package> - <name>microsoft-r-server-mro-8.0</name> - </package> - <package> - <name>microsoft-r-server-intel-mkl-8.0</name> - </package> - <package> - <name>microsoft-r-server-packages-8.0</name> - </package> - <package> - <name>microsoft-r-server-hadoop-8.0</name> - </package> - </packages> - </osSpecific> - </osSpecifics> - </service> - </services> -</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/repos/repoinfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/repos/repoinfo.xml deleted file mode 100644 index fac533d..0000000 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/repos/repoinfo.xml +++ /dev/null @@ -1,40 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<reposinfo> - <os family="redhat6"> - <repo> - <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl> - <repoid>MSFT_R-8.0</repoid> - <reponame>MSFT_R</reponame> - </repo> - </os> - <os family="redhat7"> - <repo> - <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl> - <repoid>MSFT_R-8.0</repoid> - <reponame>MSFT_R</reponame> - </repo> - </os> - <os family="suse11"> - <repo> - <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl> - <repoid>MSFT_R-8.0</repoid> - <reponame>MSFT_R</reponame> - </repo> - </os> -</reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R_SERVER/8.0.5/metainfo.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R_SERVER/8.0.5/metainfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R_SERVER/8.0.5/metainfo.xml new file mode 100644 index 0000000..9d4f920 --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R_SERVER/8.0.5/metainfo.xml @@ -0,0 +1,47 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>MICROSOFT_R_SERVER</name> + <extends>common-services/MICROSOFT_R_SERVER/8.0.5</extends> + <version>8.0.5.0</version> + <osSpecifics> + <osSpecific> + <osFamily>redhat6,redhat7,suse11</osFamily> + <packages> + <package> + <name>microsoft-r-server-mro-8.0</name> + </package> + <package> + <name>microsoft-r-server-intel-mkl-8.0</name> + </package> + <package> + <name>microsoft-r-server-packages-8.0</name> + </package> + <package> + <name>microsoft-r-server-hadoop-8.0</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R_SERVER/8.0.5/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R_SERVER/8.0.5/repos/repoinfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R_SERVER/8.0.5/repos/repoinfo.xml new file mode 100644 index 0000000..fac533d --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R_SERVER/8.0.5/repos/repoinfo.xml @@ -0,0 +1,40 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <os family="redhat6"> + <repo> + <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl> + <repoid>MSFT_R-8.0</repoid> + <reponame>MSFT_R</reponame> + </repo> + </os> + <os family="redhat7"> + <repo> + <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl> + <repoid>MSFT_R-8.0</repoid> + <reponame>MSFT_R</reponame> + </repo> + </os> + <os family="suse11"> + <repo> + <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl> + <repoid>MSFT_R-8.0</repoid> + <reponame>MSFT_R</reponame> + </repo> + </os> +</reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/61442de4/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json index eb7076c..d874db3 100644 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json @@ -1,8 +1,8 @@ { "type": "full-release", - "name": "microsoft_r-ambari.mpack", + "name": "microsoft_r_server-ambari.mpack", "version": "1.0.0.0", - "description": "Ambari Management Pack for Microsoft R", + "description": "Ambari Management Pack for Microsoft R Server", "prerequisites": { "min-ambari-version": "2.2.0.0", "min-stack-versions": [ @@ -14,17 +14,17 @@ }, "artifacts": [ { - "name": "MICROSOFT_R-common-services", + "name": "MICROSOFT_R_SERVER-common-services", "type" : "service-definitions", "source_dir" : "common-services" }, { - "name" : "MICROSOFT_R-custom-services", + "name" : "MICROSOFT_R_SERVER-custom-services", "type" : "stack-addon-service-definitions", "source_dir": "custom-services", "service_versions_map": [ { - "service_name" : "MICROSOFT_R", + "service_name" : "MICROSOFT_R_SERVER", "service_version" : "8.0.5", "applicable_stacks" : [ {