http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/files/validateYarnComponentStatus.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/files/validateYarnComponentStatus.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/files/validateYarnComponentStatus.py new file mode 100644 index 0000000..33ed8b1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/files/validateYarnComponentStatus.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import optparse +import subprocess +import json + +RESOURCEMANAGER = 'rm' +NODEMANAGER = 'nm' +HISTORYSERVER = 'hs' + +STARTED_STATE = 'STARTED' +RUNNING_STATE = 'RUNNING' + +#Return reponse for given path and address +def getResponse(path, address, ssl_enabled): + + command = "curl" + httpGssnegotiate = "--negotiate" + userpswd = "-u:" + insecure = "-k"# This is smoke test, no need to check CA of server + if ssl_enabled: + url = 'https://' + address + path + else: + url = 'http://' + address + path + + command_with_flags = [command,httpGssnegotiate,userpswd,insecure,url] + + proc = subprocess.Popen(command_with_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = proc.communicate() + response = json.loads(stdout) + if response == None: + print 'There is no response for url: ' + str(url) + raise Exception('There is no response for url: ' + str(url)) + return response + +#Verify that REST api is available for given component +def validateAvailability(component, path, addresses, ssl_enabled): + responses = {} + for address in addresses.split(','): + try: + responses[address] = getResponse(path, address, ssl_enabled) + except Exception as e: + print 'Error checking availability status of component.', e + + if not responses: + exit(1) + + is_valid = validateAvailabilityResponse(component, responses.values()[0]) + if not is_valid: + exit(1) + +#Validate component-specific response +def validateAvailabilityResponse(component, response): + try: + if component == RESOURCEMANAGER: + rm_state = response['clusterInfo']['state'] + if rm_state == STARTED_STATE: + return True + else: + print 'Resourcemanager is not started' + return False + + elif component == NODEMANAGER: + node_healthy = bool(response['nodeInfo']['nodeHealthy']) + if node_healthy: + return True + else: + return False + elif component == HISTORYSERVER: + hs_start_time = response['historyInfo']['startedOn'] + if hs_start_time > 0: + return True + else: + return False + else: + return False + except Exception as e: + print 'Error validation of availability response for ' + str(component), e + return False + +#Verify that component has required resources to work +def validateAbility(component, path, addresses, ssl_enabled): + responses = {} + for address in addresses.split(','): + try: + responses[address] = getResponse(path, address, ssl_enabled) + except Exception as e: + print 'Error checking ability of component.', e + + if not responses: + exit(1) + + is_valid = validateAbilityResponse(component, responses.values()[0]) + if not is_valid: + exit(1) + +#Validate component-specific response that it has required resources to work +def validateAbilityResponse(component, response): + try: + if component == RESOURCEMANAGER: + nodes = [] + if response.has_key('nodes') and not response['nodes'] == None and response['nodes'].has_key('node'): + nodes = response['nodes']['node'] + connected_nodes_count = len(nodes) + if connected_nodes_count == 0: + print 'There is no connected nodemanagers to resourcemanager' + return False + active_nodes = filter(lambda x: x['state'] == RUNNING_STATE, nodes) + active_nodes_count = len(active_nodes) + + if connected_nodes_count == 0: + print 'There is no connected active nodemanagers to resourcemanager' + return False + else: + return True + else: + return False + except Exception as e: + print 'Error validation of ability response', e + return False + +# +# Main. +# +def main(): + parser = optparse.OptionParser(usage="usage: %prog [options] component ") + parser.add_option("-p", "--port", dest="address", help="Host:Port for REST API of a desired component") + parser.add_option("-s", "--ssl", dest="ssl_enabled", help="Is SSL enabled for UI of component") + + (options, args) = parser.parse_args() + + component = args[0] + + address = options.address + ssl_enabled = (options.ssl_enabled) in 'true' + if component == RESOURCEMANAGER: + path = '/ws/v1/cluster/info' + elif component == NODEMANAGER: + path = '/ws/v1/node/info' + elif component == HISTORYSERVER: + path = '/ws/v1/history/info' + else: + parser.error("Invalid component") + + validateAvailability(component, path, address, ssl_enabled) + + if component == RESOURCEMANAGER: + path = '/ws/v1/cluster/nodes' + validateAbility(component, path, address, ssl_enabled) + +if __name__ == "__main__": + main()
http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/__init__.py new file mode 100644 index 0000000..35de4bb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/__init__.py @@ -0,0 +1,20 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/application_timeline_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/application_timeline_server.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/application_timeline_server.py new file mode 100644 index 0000000..c33110f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/application_timeline_server.py @@ -0,0 +1,57 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * +from yarn import yarn +from service import service + +class ApplicationTimelineServer(Script): + + def install(self, env): + self.install_packages(env) + #self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + yarn(name='apptimelineserver') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # FOR SECURITY + service('historyserver', action='start') + + def stop(self, env): + import params + env.set_params(params) + service('historyserver', action='stop') + + def status(self, env): + import status_params + env.set_params(status_params) + Execute(format("mv {yarn_historyserver_pid_file_old} {yarn_historyserver_pid_file}"), + only_if = format("test -e {yarn_historyserver_pid_file_old}", user=status_params.yarn_user)) + functions.check_process_status(status_params.yarn_historyserver_pid_file) + +if __name__ == "__main__": + ApplicationTimelineServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/historyserver.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/historyserver.py new file mode 100644 index 0000000..4184dc4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/historyserver.py @@ -0,0 +1,53 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import sys +from resource_management import * + +from yarn import yarn +from service import service + +class HistoryServer(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + yarn(name="historyserver") + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # FOR SECURITY + service('historyserver', action='start', serviceName='mapreduce') + + def stop(self, env): + import params + env.set_params(params) + service('historyserver', action='stop', serviceName='mapreduce') + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.mapred_historyserver_pid_file) + +if __name__ == "__main__": + HistoryServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/mapred_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/mapred_service_check.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/mapred_service_check.py new file mode 100644 index 0000000..f12ea61 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/mapred_service_check.py @@ -0,0 +1,80 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + +class MapReduce2ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + jar_path = format("{hadoop_mapred2_jar_location}/{hadoopMapredExamplesJarName}") + input_file = format("/user/{smokeuser}/mapredsmokeinput") + output_file = format("/user/{smokeuser}/mapredsmokeoutput") + + cleanup_cmd = format("fs -rm -r -f {output_file} {input_file}") + create_file_cmd = format("fs -put /etc/passwd {input_file}") + test_cmd = format("fs -test -e {output_file}") + run_wordcount_job = format("jar {jar_path} wordcount {input_file} {output_file}") + + log_dir = format("{mapred_log_dir_prefix}/{smokeuser}") + Directory(log_dir, owner=params.smokeuser, recursive=True) + + if params.security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};") + + Execute(kinit_cmd, + user=params.smokeuser + ) + + ExecuteHadoop(cleanup_cmd, + tries=1, + try_sleep=5, + user=params.smokeuser, + bin_dir=params.execute_path, + conf_dir=params.hadoop_conf_dir + ) + + ExecuteHadoop(create_file_cmd, + tries=1, + try_sleep=5, + user=params.smokeuser, + bin_dir=params.execute_path, + conf_dir=params.hadoop_conf_dir + ) + + ExecuteHadoop(run_wordcount_job, + tries=1, + try_sleep=5, + user=params.smokeuser, + bin_dir=params.execute_path, + conf_dir=params.hadoop_conf_dir, + logoutput=True + ) + + ExecuteHadoop(test_cmd, + user=params.smokeuser, + bin_dir=params.execute_path, + conf_dir=params.hadoop_conf_dir + ) + +if __name__ == "__main__": + MapReduce2ServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/mapreduce2_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/mapreduce2_client.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/mapreduce2_client.py new file mode 100644 index 0000000..831e5e8 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/mapreduce2_client.py @@ -0,0 +1,42 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from yarn import yarn + +class MapReduce2Client(Script): + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + yarn() + + def status(self, env): + raise ClientComponentHasNoStatus() + +if __name__ == "__main__": + MapReduce2Client().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/nodemanager.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/nodemanager.py new file mode 100644 index 0000000..8e153e0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/nodemanager.py @@ -0,0 +1,59 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from yarn import yarn +from service import service + +class Nodemanager(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + yarn(name="nodemanager") + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # FOR SECURITY + service('nodemanager', + action='start' + ) + + def stop(self, env): + import params + env.set_params(params) + + service('nodemanager', + action='stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.nodemanager_pid_file) + +if __name__ == "__main__": + Nodemanager().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/params.py new file mode 100644 index 0000000..6e7f170 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/params.py @@ -0,0 +1,176 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import os + +from resource_management import * +import status_params + +# server configurations +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +#RPM versioning support +rpm_version = default("/configurations/cluster-env/rpm_version", None) + +#hadoop params +if rpm_version: + hadoop_libexec_dir = "/usr/phd/current/hadoop-client/libexec" + hadoop_bin = "/usr/phd/current/hadoop-client/sbin" + hadoop_bin_dir = "/usr/phd/current/hadoop-client/bin" + hadoop_yarn_home = '/usr/phd/current/hadoop-yarn-client' + hadoop_mapred2_jar_location = '/usr/phd/current/hadoop-mapreduce-client' + mapred_bin = '/usr/phd/current/hadoop-mapreduce-client/sbin' + yarn_bin = '/usr/phd/current/hadoop-yarn-client/sbin' + yarn_container_bin = '/usr/phd/current/hadoop-yarn-client/bin' +else: + hadoop_libexec_dir = "/usr/lib/hadoop/libexec" + hadoop_bin = "/usr/lib/hadoop/sbin" + hadoop_bin_dir = "/usr/bin" + hadoop_yarn_home = '/usr/lib/hadoop-yarn' + hadoop_mapred2_jar_location = "/usr/lib/hadoop-mapreduce" + mapred_bin = "/usr/lib/hadoop-mapreduce/sbin" + yarn_bin = "/usr/lib/hadoop-yarn/sbin" + yarn_container_bin = "/usr/lib/hadoop-yarn/bin" + +hadoop_conf_dir = "/etc/hadoop/conf" +limits_conf_dir = "/etc/security/limits.d" +execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir + os.pathsep + yarn_container_bin + +ulimit_cmd = "ulimit -c unlimited;" + +mapred_user = status_params.mapred_user +yarn_user = status_params.yarn_user +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] + +smokeuser = config['configurations']['cluster-env']['smokeuser'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group'] +kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +rm_hosts = config['clusterHostInfo']['rm_host'] +rm_host = rm_hosts[0] +rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1] +rm_https_port = "8090" +# TODO UPGRADE default, update site during upgrade +rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") + +java64_home = config['hostLevelParams']['java_home'] +hadoop_ssl_enabled = default("/configurations/core-site/hadoop.ssl.enabled", False) + +yarn_heapsize = config['configurations']['yarn-env']['yarn_heapsize'] +resourcemanager_heapsize = config['configurations']['yarn-env']['resourcemanager_heapsize'] +nodemanager_heapsize = config['configurations']['yarn-env']['nodemanager_heapsize'] +apptimelineserver_heapsize = default("/configurations/yarn-env/apptimelineserver_heapsize", 1024) +ats_leveldb_dir = config['configurations']['yarn-site']['yarn.timeline-service.leveldb-timeline-store.path'] +yarn_log_dir_prefix = config['configurations']['yarn-env']['yarn_log_dir_prefix'] +yarn_pid_dir_prefix = status_params.yarn_pid_dir_prefix +mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix +mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_prefix'] +mapred_env_sh_template = config['configurations']['mapred-env']['content'] +yarn_env_sh_template = config['configurations']['yarn-env']['content'] + +if len(rm_hosts) > 1: + additional_rm_host = rm_hosts[1] + rm_webui_address = format("{rm_host}:{rm_port},{additional_rm_host}:{rm_port}") + rm_webui_https_address = format("{rm_host}:{rm_https_port},{additional_rm_host}:{rm_https_port}") +else: + rm_webui_address = format("{rm_host}:{rm_port}") + rm_webui_https_address = format("{rm_host}:{rm_https_port}") + +nm_webui_address = config['configurations']['yarn-site']['yarn.nodemanager.webapp.address'] +hs_webui_address = config['configurations']['mapred-site']['mapreduce.jobhistory.webapp.address'] + +nm_local_dirs = config['configurations']['yarn-site']['yarn.nodemanager.local-dirs'] +nm_log_dirs = config['configurations']['yarn-site']['yarn.nodemanager.log-dirs'] + +distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar" +hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" + +yarn_pid_dir = status_params.yarn_pid_dir +mapred_pid_dir = status_params.mapred_pid_dir + +mapred_log_dir = format("{mapred_log_dir_prefix}/{mapred_user}") +yarn_log_dir = format("{yarn_log_dir_prefix}/{yarn_user}") +mapred_job_summary_log = format("{mapred_log_dir_prefix}/{mapred_user}/hadoop-mapreduce.jobsummary.log") +yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduce.jobsummary.log") + +user_group = config['configurations']['cluster-env']['user_group'] + +#exclude file +exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) +exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") + +hostname = config['hostname'] + +ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) +has_ats = not len(ats_host) == 0 + +if security_enabled: + _rm_principal_name = config['configurations']['yarn-site']['yarn.resourcemanager.principal'] + _rm_keytab = config['configurations']['yarn-site']['yarn.resourcemanager.keytab'] + _rm_principal_name = _rm_principal_name.replace('_HOST',hostname.lower()) + + rm_kinit_cmd = format("{kinit_path_local} -kt {_rm_keytab} {_rm_principal_name};") + + # YARN timeline security options are only available in PHD Champlain + if has_ats: + _yarn_timelineservice_principal_name = config['configurations']['yarn-site']['yarn.timeline-service.principal'] + _yarn_timelineservice_principal_name = _yarn_timelineservice_principal_name.replace('_HOST', hostname.lower()) + _yarn_timelineservice_keytab = config['configurations']['yarn-site']['yarn.timeline-service.keytab'] + yarn_timelineservice_kinit_cmd = format("{kinit_path_local} -kt {_yarn_timelineservice_keytab} {_yarn_timelineservice_principal_name};") +else: + rm_kinit_cmd = "" + yarn_timelineservice_kinit_cmd = "" + +yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-aggregation-enable'] +yarn_nm_app_log_dir = config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir'] +mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir'] +mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir'] +jobhistory_heapsize = default("/configurations/mapred-env/jobhistory_heapsize", "900") + +#for create_hdfs_directory +hostname = config["hostname"] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +import functools +#create partial functions with common arguments for every HdfsDirectory call +#to create hdfs directory we need to call params.HdfsDirectory in code +HdfsDirectory = functools.partial( + HdfsDirectory, + conf_dir=hadoop_conf_dir, + hdfs_user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + bin_dir = hadoop_bin_dir +) +update_exclude_file_only = config['commandParams']['update_exclude_file_only'] + +mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) + +#taskcontroller.cfg + +mapred_local_dir = "/tmp/hadoop-mapred/mapred/local" +hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] +min_user_id = config['configurations']['yarn-env']['min_user_id'] + http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/resourcemanager.py new file mode 100644 index 0000000..4d40d68 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/resourcemanager.py @@ -0,0 +1,101 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from yarn import yarn +from service import service + + +class Resourcemanager(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + + env.set_params(params) + yarn(name='resourcemanager') + + def start(self, env): + import params + + env.set_params(params) + self.configure(env) # FOR SECURITY + service('resourcemanager', + action='start' + ) + + def stop(self, env): + import params + + env.set_params(params) + + service('resourcemanager', + action='stop' + ) + + def status(self, env): + import status_params + + env.set_params(status_params) + check_process_status(status_params.resourcemanager_pid_file) + pass + + def refreshqueues(self, env): + import params + + self.configure(env) + env.set_params(params) + + service('resourcemanager', + action='refreshQueues' + ) + + def decommission(self, env): + import params + + env.set_params(params) + rm_kinit_cmd = params.rm_kinit_cmd + yarn_user = params.yarn_user + conf_dir = params.hadoop_conf_dir + user_group = params.user_group + + yarn_refresh_cmd = format("{rm_kinit_cmd} yarn --config {conf_dir} rmadmin -refreshNodes") + + File(params.exclude_file_path, + content=Template("exclude_hosts_list.j2"), + owner=yarn_user, + group=user_group + ) + + if params.update_exclude_file_only == False: + Execute(yarn_refresh_cmd, + environment= {'PATH' : params.execute_path }, + user=yarn_user) + pass + pass + + +if __name__ == "__main__": + Resourcemanager().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/service.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/service.py new file mode 100644 index 0000000..466f637 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/service.py @@ -0,0 +1,75 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + + +def service(componentName, action='start', serviceName='yarn'): + + import params + + if serviceName == 'mapreduce' and componentName == 'historyserver': + daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh") + pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid") + usr = params.mapred_user + else: + daemon = format("{yarn_bin}/yarn-daemon.sh") + pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-{componentName}.pid") + usr = params.yarn_user + + cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}") + + if action == 'start': + daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}") + check_process = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + + # Remove the pid file if its corresponding process is not running. + File(pid_file, + action="delete", + not_if=check_process) + + # Attempt to start the process. Internally, this is skipped if the process is already running. + Execute(daemon_cmd, + user=usr, + not_if=check_process + ) + + # Ensure that the process with the expected PID exists. + Execute(check_process, + user=usr, + not_if=check_process, + initial_wait=5 + ) + + elif action == 'stop': + daemon_cmd = format("{cmd} stop {componentName}") + Execute(daemon_cmd, + user=usr) + + File(pid_file, + action="delete") + + elif action == 'refreshQueues': + refresh_cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_container_bin}/yarn rmadmin -refreshQueues") + + Execute(refresh_cmd, + user=usr, + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/service_check.py new file mode 100644 index 0000000..7189664 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/service_check.py @@ -0,0 +1,68 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +import sys + +class ServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + run_yarn_check_cmd = format("yarn --config {hadoop_conf_dir} node -list") + + component_type = 'rm' + if params.hadoop_ssl_enabled: + component_address = params.rm_webui_https_address + else: + component_address = params.rm_webui_address + + validateStatusFileName = "validateYarnComponentStatus.py" + validateStatusFilePath = format("{tmp_dir}/{validateStatusFileName}") + python_executable = sys.executable + validateStatusCmd = format("{python_executable} {validateStatusFilePath} {component_type} -p {component_address} -s {hadoop_ssl_enabled}") + + if params.security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};") + smoke_cmd = format("{kinit_cmd} {validateStatusCmd}") + else: + smoke_cmd = validateStatusCmd + + File(validateStatusFilePath, + content=StaticFile(validateStatusFileName), + mode=0755 + ) + + Execute(smoke_cmd, + tries=3, + try_sleep=5, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=params.smokeuser, + logoutput=True + ) + + Execute(run_yarn_check_cmd, + path=params.execute_path, + user=params.smokeuser + ) + +if __name__ == "__main__": + ServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/status_params.py new file mode 100644 index 0000000..498a885 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/status_params.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +config = Script.get_config() + +mapred_user = config['configurations']['mapred-env']['mapred_user'] +yarn_user = config['configurations']['yarn-env']['yarn_user'] +yarn_pid_dir_prefix = config['configurations']['yarn-env']['yarn_pid_dir_prefix'] +mapred_pid_dir_prefix = config['configurations']['mapred-env']['mapred_pid_dir_prefix'] +yarn_pid_dir = format("{yarn_pid_dir_prefix}/{yarn_user}") +mapred_pid_dir = format("{mapred_pid_dir_prefix}/{mapred_user}") + +resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanager.pid") +nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid") +yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid") +yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated +mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/yarn.py new file mode 100644 index 0000000..df3a9d2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/yarn.py @@ -0,0 +1,238 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +import sys +import os + + +def yarn(name = None): + import params + + + if name in ["nodemanager","historyserver"]: + if params.yarn_log_aggregation_enabled: + params.HdfsDirectory(params.yarn_nm_app_log_dir, + action="create_delayed", + owner=params.yarn_user, + group=params.user_group, + mode=0777, + recursive_chmod=True + ) + params.HdfsDirectory("/mapred", + action="create_delayed", + owner=params.mapred_user + ) + params.HdfsDirectory("/mapred/system", + action="create_delayed", + owner=params.hdfs_user + ) + params.HdfsDirectory(params.mapreduce_jobhistory_intermediate_done_dir, + action="create_delayed", + owner=params.mapred_user, + group=params.user_group, + mode=0777 + ) + + params.HdfsDirectory(params.mapreduce_jobhistory_done_dir, + action="create_delayed", + owner=params.mapred_user, + group=params.user_group, + mode=01777 + ) + params.HdfsDirectory(None, action="create") + + if name == "nodemanager": + Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','), + owner=params.yarn_user, + recursive=True, + ignore_failures=True, + ) + + Directory([params.yarn_pid_dir, params.yarn_log_dir], + owner=params.yarn_user, + group=params.user_group, + recursive=True + ) + + Directory([params.mapred_pid_dir, params.mapred_log_dir], + owner=params.mapred_user, + group=params.user_group, + recursive=True + ) + Directory([params.yarn_log_dir_prefix], + owner=params.yarn_user, + recursive=True, + ignore_failures=True, + ) + + XmlConfig("core-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['core-site'], + configuration_attributes=params.config['configuration_attributes']['core-site'], + owner=params.hdfs_user, + group=params.user_group, + mode=0644 + ) + + XmlConfig("mapred-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['mapred-site'], + configuration_attributes=params.config['configuration_attributes']['mapred-site'], + owner=params.yarn_user, + group=params.user_group, + mode=0644 + ) + + XmlConfig("yarn-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['yarn-site'], + configuration_attributes=params.config['configuration_attributes']['yarn-site'], + owner=params.yarn_user, + group=params.user_group, + mode=0644 + ) + + XmlConfig("capacity-scheduler.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['capacity-scheduler'], + configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'], + owner=params.yarn_user, + group=params.user_group, + mode=0644 + ) + + if name == 'resourcemanager': + File(params.yarn_job_summary_log, + owner=params.yarn_user, + group=params.user_group + ) + elif name == 'apptimelineserver': + Directory(params.ats_leveldb_dir, + owner=params.yarn_user, + group=params.user_group, + recursive=True + ) + + File(params.rm_nodes_exclude_path, + owner=params.yarn_user, + group=params.user_group + ) + + File(format("{limits_conf_dir}/yarn.conf"), + mode=0644, + content=Template('yarn.conf.j2') + ) + + File(format("{limits_conf_dir}/mapreduce.conf"), + mode=0644, + content=Template('mapreduce.conf.j2') + ) + + File(format("{hadoop_conf_dir}/yarn-env.sh"), + owner=params.yarn_user, + group=params.user_group, + mode=0755, + content=InlineTemplate(params.yarn_env_sh_template) + ) + + if params.security_enabled: + container_executor = format("{yarn_container_bin}/container-executor") + File(container_executor, + group=params.yarn_executor_container_group, + mode=06050 + ) + + File(format("{hadoop_conf_dir}/container-executor.cfg"), + group=params.user_group, + mode=0644, + content=Template('container-executor.cfg.j2') + ) + + + if params.security_enabled: + tc_mode = 0644 + tc_owner = "root" + else: + tc_mode = None + tc_owner = params.hdfs_user + + File(format("{hadoop_conf_dir}/mapred-env.sh"), + owner=tc_owner, + content=InlineTemplate(params.mapred_env_sh_template) + ) + + if params.security_enabled: + File(os.path.join(params.hadoop_bin, "task-controller"), + owner="root", + group=params.mapred_tt_group, + mode=06050 + ) + File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'), + owner = tc_owner, + mode = tc_mode, + group = params.mapred_tt_group, + content=Template("taskcontroller.cfg.j2") + ) + else: + File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'), + owner=tc_owner, + content=Template("taskcontroller.cfg.j2") + ) + + if "mapred-site" in params.config['configurations']: + XmlConfig("mapred-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['mapred-site'], + configuration_attributes=params.config['configuration_attributes']['mapred-site'], + owner=params.mapred_user, + group=params.user_group + ) + + if "capacity-scheduler" in params.config['configurations']: + XmlConfig("capacity-scheduler.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations'][ + 'capacity-scheduler'], + configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'], + owner=params.hdfs_user, + group=params.user_group + ) + + if os.path.exists(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml')): + File(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml'), + owner=params.mapred_user, + group=params.user_group + ) + + if os.path.exists( + os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example')): + File(os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example'), + owner=params.mapred_user, + group=params.user_group + ) + + if os.path.exists( + os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example')): + File(os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example'), + owner=params.mapred_user, + group=params.user_group + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/yarn_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/yarn_client.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/yarn_client.py new file mode 100644 index 0000000..f6b4b44 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/scripts/yarn_client.py @@ -0,0 +1,42 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from yarn import yarn + +class YarnClient(Script): + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + yarn() + + def status(self, env): + raise ClientComponentHasNoStatus() + +if __name__ == "__main__": + YarnClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/container-executor.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/container-executor.cfg.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/container-executor.cfg.j2 new file mode 100644 index 0000000..c6f1ff6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/container-executor.cfg.j2 @@ -0,0 +1,40 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +yarn.nodemanager.local-dirs={{nm_local_dirs}} +yarn.nodemanager.log-dirs={{nm_log_dirs}} +yarn.nodemanager.linux-container-executor.group={{yarn_executor_container_group}} +banned.users=hdfs,yarn,mapred,bin +min.user.id={{min_user_id}} http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/exclude_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/exclude_hosts_list.j2 new file mode 100644 index 0000000..c7ce416 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/exclude_hosts_list.j2 @@ -0,0 +1,21 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in exclude_hosts %} +{{host}} +{% endfor %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/mapreduce.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/mapreduce.conf.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/mapreduce.conf.j2 new file mode 100644 index 0000000..b996645 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/mapreduce.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{mapred_user}} - nofile 32768 +{{mapred_user}} - nproc 65536 http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/taskcontroller.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/taskcontroller.cfg.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/taskcontroller.cfg.j2 new file mode 100644 index 0000000..3d5f4f2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/taskcontroller.cfg.j2 @@ -0,0 +1,38 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +mapred.local.dir={{mapred_local_dir}} +mapreduce.tasktracker.group={{mapred_tt_group}} +hadoop.log.dir={{hdfs_log_dir_prefix}}/{{mapred_user}} http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/yarn.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/yarn.conf.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/yarn.conf.j2 new file mode 100644 index 0000000..3bd7a45 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/YARN/package/templates/yarn.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{yarn_user}} - nofile 32768 +{{yarn_user}} - nproc 65536 http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/configuration/zookeeper-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/configuration/zookeeper-env.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/configuration/zookeeper-env.xml new file mode 100644 index 0000000..4c305b0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/configuration/zookeeper-env.xml @@ -0,0 +1,85 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>zk_user</name> + <value>zookeeper</value> + <property-type>USER</property-type> + <description>ZooKeeper User.</description> + </property> + <property> + <name>zk_data_dir</name> + <value>/hadoop/zookeeper</value> + <description>Data directory for ZooKeeper.</description> + </property> + <property> + <name>zk_log_dir</name> + <value>/var/log/zookeeper</value> + <description>ZooKeeper Log Dir</description> + </property> + <property> + <name>zk_pid_dir</name> + <value>/var/run/zookeeper</value> + <description>ZooKeeper Pid Dir</description> + </property> + <property> + <name>tickTime</name> + <value>2000</value> + <description>The length of a single tick in milliseconds, which is the basic time unit used by ZooKeeper</description> + </property> + <property> + <name>initLimit</name> + <value>10</value> + <description>Ticks to allow for sync at Init.</description> + </property> + <property> + <name>syncLimit</name> + <value>5</value> + <description>Ticks to allow for sync at Runtime.</description> + </property> + <property> + <name>clientPort</name> + <value>2181</value> + <description>Port for running ZK Server.</description> + </property> + + <!-- zookeeper-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for zookeeper-env.sh file</description> + <value> +export JAVA_HOME={{java64_home}} +export ZOOKEEPER_HOME={{zk_home}} +export ZOO_LOG_DIR={{zk_log_dir}} +export ZOOPIDFILE={{zk_pid_file}} +export SERVER_JVMFLAGS={{zk_server_heapsize}} +export JAVA=$JAVA_HOME/bin/java +export CLASSPATH=$CLASSPATH:/usr/share/zookeeper/* + +{% if security_enabled %} +export SERVER_JVMFLAGS="$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}" +export CLIENT_JVMFLAGS="$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}" +{% endif %} + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/configuration/zookeeper-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/configuration/zookeeper-log4j.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/configuration/zookeeper-log4j.xml new file mode 100644 index 0000000..6fcf5bc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/configuration/zookeeper-log4j.xml @@ -0,0 +1,101 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + + <property> + <name>content</name> + <description>Custom log4j.properties</description> + <value> +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# + +# +# ZooKeeper Logging Configuration +# + +# DEFAULT: console appender only +log4j.rootLogger=INFO, CONSOLE + +# Example with rolling log file +#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE + +# Example with rolling log file and tracing +#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE + +# +# Log INFO level and above messages to the console +# +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +# +# Add ROLLINGFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLINGFILE.Threshold=DEBUG +log4j.appender.ROLLINGFILE.File=zookeeper.log + +# Max log file size of 10MB +log4j.appender.ROLLINGFILE.MaxFileSize=10MB +# uncomment the next line to limit number of backup files +#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 + +log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + + +# +# Add TRACEFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.TRACEFILE=org.apache.log4j.FileAppender +log4j.appender.TRACEFILE.Threshold=TRACE +log4j.appender.TRACEFILE.File=zookeeper_trace.log + +log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout +### Notice we are including log4j's NDC here (%x) +log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/metainfo.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/metainfo.xml new file mode 100644 index 0000000..f304c83 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/metainfo.xml @@ -0,0 +1,89 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>ZOOKEEPER</name> + <displayName>ZooKeeper</displayName> + <comment>Centralized service which provides highly reliable distributed coordination</comment> + <version>3.4.5.phd.3.0.0.0</version> + <components> + + <component> + <name>ZOOKEEPER_SERVER</name> + <displayName>ZooKeeper Server</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/zookeeper_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>ZOOKEEPER_CLIENT</name> + <displayName>ZooKeeper Client</displayName> + <category>CLIENT</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/zookeeper_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + <configFiles> + <configFile> + <type>env</type> + <fileName>zookeeper-env.sh</fileName> + <dictionaryName>zookeeper-env</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>log4j.properties</fileName> + <dictionaryName>zookeeper-log4j</dictionaryName> + </configFile> + </configFiles> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>zookeeper</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <configuration-dependencies> + <config-type>zookeeper-log4j</config-type> + <config-type>zookeeper-env</config-type> + <config-type>zoo.cfg</config-type> + </configuration-dependencies> + <restartRequiredAfterChange>true</restartRequiredAfterChange> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkEnv.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkEnv.sh b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkEnv.sh new file mode 100644 index 0000000..fa1b832 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkEnv.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script should be sourced into other zookeeper +# scripts to setup the env variables + +# We use ZOOCFGDIR if defined, +# otherwise we use /etc/zookeeper +# or the conf directory that is +# a sibling of this script's directory +if [ "x$ZOOCFGDIR" = "x" ] +then + if [ -d "/etc/zookeeper" ] + then + ZOOCFGDIR="/etc/zookeeper" + else + ZOOCFGDIR="$ZOOBINDIR/../conf" + fi +fi + +if [ "x$ZOOCFG" = "x" ] +then + ZOOCFG="zoo.cfg" +fi + +ZOOCFG="$ZOOCFGDIR/$ZOOCFG" + +if [ -e "$ZOOCFGDIR/zookeeper-env.sh" ] +then + . "$ZOOCFGDIR/zookeeper-env.sh" +fi + +if [ "x${ZOO_LOG_DIR}" = "x" ] +then + ZOO_LOG_DIR="." +fi + +if [ "x${ZOO_LOG4J_PROP}" = "x" ] +then + ZOO_LOG4J_PROP="INFO,CONSOLE" +fi + +#add the zoocfg dir to classpath +CLASSPATH="$ZOOCFGDIR:$CLASSPATH" + +for i in "$ZOOBINDIR"/../src/java/lib/*.jar +do + CLASSPATH="$i:$CLASSPATH" +done + +#make it work in the release +for i in "$ZOOBINDIR"/../lib/*.jar +do + CLASSPATH="$i:$CLASSPATH" +done + +#make it work in the release +for i in "$ZOOBINDIR"/../zookeeper-*.jar +do + CLASSPATH="$i:$CLASSPATH" +done + +#make it work for developers +for d in "$ZOOBINDIR"/../build/lib/*.jar +do + CLASSPATH="$d:$CLASSPATH" +done + +#make it work for developers +CLASSPATH="$ZOOBINDIR/../build/classes:$CLASSPATH" + +case "`uname`" in + CYGWIN*) cygwin=true ;; + *) cygwin=false ;; +esac + +if $cygwin +then + CLASSPATH=`cygpath -wp "$CLASSPATH"` +fi + +#echo "CLASSPATH=$CLASSPATH" http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkServer.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkServer.sh b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkServer.sh new file mode 100644 index 0000000..dd75a58 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkServer.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# If this scripted is run out of /usr/bin or some other system bin directory +# it should be linked to and not copied. Things like java jar files are found +# relative to the canonical path of this script. +# + +# See the following page for extensive details on setting +# up the JVM to accept JMX remote management: +# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# by default we allow local JMX connections +if [ "x$JMXLOCALONLY" = "x" ] +then + JMXLOCALONLY=false +fi + +if [ "x$JMXDISABLE" = "x" ] +then + echo "JMX enabled by default" + # for some reason these two options are necessary on jdk6 on Ubuntu + # accord to the docs they are not necessary, but otw jconsole cannot + # do a local attach + ZOOMAIN="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY org.apache.zookeeper.server.quorum.QuorumPeerMain" +else + echo "JMX disabled by user request" + ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain" +fi + +# Only follow symlinks if readlink supports it +if readlink -f "$0" > /dev/null 2>&1 +then + ZOOBIN=`readlink -f "$0"` +else + ZOOBIN="$0" +fi +ZOOBINDIR=`dirname "$ZOOBIN"` + +. "$ZOOBINDIR"/zkEnv.sh + +if [ "x$2" != "x" ] +then + ZOOCFG="$ZOOCFGDIR/$2" +fi + +if $cygwin +then + ZOOCFG=`cygpath -wp "$ZOOCFG"` + # cygwin has a "kill" in the shell itself, gets confused + KILL=/bin/kill +else + KILL=kill +fi + +echo "Using config: $ZOOCFG" + +ZOOPIDFILE=$(grep dataDir "$ZOOCFG" | sed -e 's/.*=//')/zookeeper_server.pid + + +case $1 in +start) + echo "Starting zookeeper ... " + $JAVA "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \ + -cp "$CLASSPATH" $JVMFLAGS $ZOOMAIN "$ZOOCFG" & + /bin/echo -n $! > "$ZOOPIDFILE" + echo STARTED + ;; +stop) + echo "Stopping zookeeper ... " + if [ ! -f "$ZOOPIDFILE" ] + then + echo "error: could not find file $ZOOPIDFILE" + exit 1 + else + $KILL -9 $(cat "$ZOOPIDFILE") + rm "$ZOOPIDFILE" + echo STOPPED + fi + ;; +upgrade) + shift + echo "upgrading the servers to 3.*" + java "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \ + -cp "$CLASSPATH" $JVMFLAGS org.apache.zookeeper.server.upgrade.UpgradeMain ${@} + echo "Upgrading ... " + ;; +restart) + shift + "$0" stop ${@} + sleep 3 + "$0" start ${@} + ;; +status) + STAT=`echo stat | nc localhost $(grep clientPort "$ZOOCFG" | sed -e 's/.*=//') 2> /dev/null| grep Mode` + if [ "x$STAT" = "x" ] + then + echo "Error contacting service. It is probably not running." + else + echo $STAT + fi + ;; +*) + echo "Usage: $0 {start|stop|restart|status}" >&2 + +esac http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkService.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkService.sh b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkService.sh new file mode 100644 index 0000000..6e167a4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkService.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +zkcli_script=$1 +user=$2 +conf_dir=$3 +su -s /bin/bash - $user -c "source $conf_dir/zookeeper-env.sh ; echo 'ls /' | $zkcli_script" http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkSmoke.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkSmoke.sh b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkSmoke.sh new file mode 100644 index 0000000..f31a6ea --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/files/zkSmoke.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +smoke_script=$1 +smoke_user=$2 +conf_dir=$3 +client_port=$4 +security_enabled=$5 +kinit_path_local=$6 +smoke_user_keytab=$7 +export ZOOKEEPER_EXIT_CODE=0 +test_output_file=/tmp/zkSmoke.out +errors_expr="ERROR|Exception" +acceptable_expr="SecurityException" +zkhosts=` grep "^server\.[[:digit:]]" $conf_dir/zoo.cfg | cut -f 2 -d '=' | cut -f 1 -d ':' | tr '\n' ' ' ` +zk_node1=`echo $zkhosts | tr ' ' '\n' | head -n 1` +echo "zk_node1=$zk_node1" +if [[ $security_enabled == "True" ]]; then + kinitcmd="$kinit_path_local -kt $smoke_user_keytab $smoke_user" + su -s /bin/bash - $smoke_user -c "$kinitcmd" +fi + +function verify_output() { + if [ -f $test_output_file ]; then + errors=`grep -E $errors_expr $test_output_file | grep -v $acceptable_expr` + if [ "$?" -eq 0 ]; then + echo "Error found in the zookeeper smoke test. Exiting." + echo $errors + exit 1 + fi + fi +} + +# Delete /zk_smoketest znode if exists +su -s /bin/bash - $smoke_user -c "source $conf_dir/zookeeper-env.sh ; echo delete /zk_smoketest | ${smoke_script} -server $zk_node1:$client_port" 2>&1>$test_output_file +# Create /zk_smoketest znode on one zookeeper server +su -s /bin/bash - $smoke_user -c "source $conf_dir/zookeeper-env.sh ; echo create /zk_smoketest smoke_data | ${smoke_script} -server $zk_node1:$client_port" 2>&1>>$test_output_file +verify_output + +for i in $zkhosts ; do + echo "Running test on host $i" + # Verify the data associated with znode across all the nodes in the zookeeper quorum + su -s /bin/bash - $smoke_user -c "source $conf_dir/zookeeper-env.sh ; echo 'get /zk_smoketest' | ${smoke_script} -server $i:$client_port" + su -s /bin/bash - $smoke_user -c "source $conf_dir/zookeeper-env.sh ; echo 'ls /' | ${smoke_script} -server $i:$client_port" + output=$(su -s /bin/bash - $smoke_user -c "source $conf_dir/zookeeper-env.sh ; echo 'get /zk_smoketest' | ${smoke_script} -server $i:$client_port") + echo $output | grep smoke_data + if [[ $? -ne 0 ]] ; then + echo "Data associated with znode /zk_smoketests is not consistent on host $i" + ((ZOOKEEPER_EXIT_CODE=$ZOOKEEPER_EXIT_CODE+1)) + fi +done + +su -s /bin/bash - $smoke_user -c "source $conf_dir/zookeeper-env.sh ; echo 'delete /zk_smoketest' | ${smoke_script} -server $zk_node1:$client_port" +if [[ "$ZOOKEEPER_EXIT_CODE" -ne "0" ]] ; then + echo "Zookeeper Smoke Test: Failed" +else + echo "Zookeeper Smoke Test: Passed" +fi +exit $ZOOKEEPER_EXIT_CODE http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/__init__.py new file mode 100644 index 0000000..35de4bb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/__init__.py @@ -0,0 +1,20 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +"""
