http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/params.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/params.py new file mode 100644 index 0000000..9fa99b5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/params.py @@ -0,0 +1,86 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +import status_params + +# server configurations +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +#RPM versioning support +rpm_version = default("/configurations/cluster-env/rpm_version", None) + +#hadoop params +if rpm_version: + zk_home = '/usr/phd/current/zookeeper-client' + zk_bin = '/usr/phd/current/zookeeper-client/bin' + smoke_script = '/usr/phd/current/zookeeper-client/bin/zkCli.sh' +else: + zk_home = '/usr' + zk_bin = '/usr/lib/zookeeper/bin' + smoke_script = "/usr/lib/zookeeper/bin/zkCli.sh" + +config_dir = "/etc/zookeeper/conf" +zk_user = config['configurations']['zookeeper-env']['zk_user'] +hostname = config['hostname'] +user_group = config['configurations']['cluster-env']['user_group'] +zk_env_sh_template = config['configurations']['zookeeper-env']['content'] + +zk_log_dir = config['configurations']['zookeeper-env']['zk_log_dir'] +zk_data_dir = config['configurations']['zookeeper-env']['zk_data_dir'] +zk_pid_dir = status_params.zk_pid_dir +zk_pid_file = status_params.zk_pid_file +zk_server_heapsize = "-Xmx1024m" + +tickTime = config['configurations']['zookeeper-env']['tickTime'] +initLimit = config['configurations']['zookeeper-env']['initLimit'] +syncLimit = config['configurations']['zookeeper-env']['syncLimit'] +clientPort = config['configurations']['zookeeper-env']['clientPort'] + +if 'zoo.cfg' in config['configurations']: + zoo_cfg_properties_map = config['configurations']['zoo.cfg'] +else: + zoo_cfg_properties_map = {} +zoo_cfg_properties_map_length = len(zoo_cfg_properties_map) + +zk_principal_name = default("/configurations/zookeeper-env/zookeeper_principal_name", "zookee...@example.com") +zk_principal = zk_principal_name.replace('_HOST',hostname.lower()) + +java64_home = config['hostLevelParams']['java_home'] + +zookeeper_hosts = config['clusterHostInfo']['zookeeper_hosts'] +zookeeper_hosts.sort() + +zk_keytab_path = config['configurations']['zookeeper-env']['zookeeper_keytab_path'] +zk_server_jaas_file = format("{config_dir}/zookeeper_jaas.conf") +zk_client_jaas_file = format("{config_dir}/zookeeper_client_jaas.conf") +security_enabled = config['configurations']['cluster-env']['security_enabled'] + +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) + +#log4j.properties +if (('zookeeper-log4j' in config['configurations']) and ('content' in config['configurations']['zookeeper-log4j'])): + log4j_props = config['configurations']['zookeeper-log4j']['content'] +else: + log4j_props = None
http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/service_check.py new file mode 100644 index 0000000..87c13db --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/service_check.py @@ -0,0 +1,46 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + +class ZookeeperServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + File(format("{tmp_dir}/zkSmoke.sh"), + mode=0755, + content=StaticFile('zkSmoke.sh') + ) + + cmd_qourum = format("{tmp_dir}/zkSmoke.sh {smoke_script} {smokeuser} {config_dir} {clientPort} " + "{security_enabled} {kinit_path_local} {smokeUserKeytab}", + smokeUserKeytab=params.smoke_user_keytab if params.security_enabled else "no_keytab") + + Execute(cmd_qourum, + tries=3, + try_sleep=5, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + logoutput=True + ) + +if __name__ == "__main__": + ZookeeperServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/status_params.py new file mode 100644 index 0000000..36c5c30 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/status_params.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +config = Script.get_config() + +zk_pid_dir = config['configurations']['zookeeper-env']['zk_pid_dir'] +zk_pid_file = format("{zk_pid_dir}/zookeeper_server.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper.py new file mode 100644 index 0000000..4f2bb1f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper.py @@ -0,0 +1,110 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import os + +from resource_management import * +import sys + + +def zookeeper(type = None): + import params + + Directory(params.config_dir, + owner=params.zk_user, + recursive=True, + group=params.user_group + ) + + File(format("{config_dir}/zookeeper-env.sh"), + content=InlineTemplate(params.zk_env_sh_template), + owner=params.zk_user, + group=params.user_group + ) + + + configFile("zoo.cfg", template_name="zoo.cfg.j2") + configFile("configuration.xsl", template_name="configuration.xsl.j2") + + Directory(params.zk_pid_dir, + owner=params.zk_user, + recursive=True, + group=params.user_group + ) + + Directory(params.zk_log_dir, + owner=params.zk_user, + recursive=True, + group=params.user_group + ) + + Directory(params.zk_data_dir, + owner=params.zk_user, + recursive=True, + group=params.user_group + ) + + if type == 'server': + myid = str(sorted(params.zookeeper_hosts).index(params.hostname) + 1) + + File(format("{zk_data_dir}/myid"), + mode = 0644, + content = myid + ) + + if (params.log4j_props != None): + File(format("{params.config_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.zk_user, + content=params.log4j_props + ) + elif (os.path.exists(format("{params.config_dir}/log4j.properties"))): + File(format("{params.config_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.zk_user + ) + + if params.security_enabled: + if type == "server": + configFile("zookeeper_jaas.conf", template_name="zookeeper_jaas.conf.j2") + configFile("zookeeper_client_jaas.conf", template_name="zookeeper_client_jaas.conf.j2") + else: + configFile("zookeeper_client_jaas.conf", template_name="zookeeper_client_jaas.conf.j2") + + File(format("{config_dir}/zoo_sample.cfg"), + owner=params.zk_user, + group=params.user_group + ) + + +def configFile(name, template_name=None): + import params + + File(format("{config_dir}/{name}"), + content=Template(template_name), + owner=params.zk_user, + group=params.user_group + ) + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_client.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_client.py new file mode 100644 index 0000000..4bffac3 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_client.py @@ -0,0 +1,42 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from zookeeper import zookeeper + +class ZookeeperClient(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + + zookeeper(type='client') + + def status(self, env): + raise ClientComponentHasNoStatus() + +if __name__ == "__main__": + ZookeeperClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_server.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_server.py new file mode 100644 index 0000000..9b9112c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_server.py @@ -0,0 +1,54 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from zookeeper import zookeeper +from zookeeper_service import zookeeper_service + +class ZookeeperServer(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + def configure(self, env): + import params + env.set_params(params) + zookeeper(type='server') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + zookeeper_service(action = 'start') + + def stop(self, env): + import params + env.set_params(params) + zookeeper_service(action = 'stop') + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.zk_pid_file) + +if __name__ == "__main__": + ZookeeperServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_service.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_service.py new file mode 100644 index 0000000..d642bc6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/scripts/zookeeper_service.py @@ -0,0 +1,42 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + +def zookeeper_service(action='start'): + import params + + cmd = format("env ZOOCFGDIR={config_dir} ZOOCFG=zoo.cfg {zk_bin}/zkServer.sh") + + if action == 'start': + daemon_cmd = format("source {config_dir}/zookeeper-env.sh ; {cmd} start") + no_op_test = format("ls {zk_pid_file} >/dev/null 2>&1 && ps `cat {zk_pid_file}` >/dev/null 2>&1") + Execute(daemon_cmd, + not_if=no_op_test, + user=params.zk_user + ) + elif action == 'stop': + daemon_cmd = format("source {config_dir}/zookeeper-env.sh ; {cmd} stop") + rm_pid = format("rm -f {zk_pid_file}") + Execute(daemon_cmd, + user=params.zk_user + ) + Execute(rm_pid) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/configuration.xsl.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/configuration.xsl.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/configuration.xsl.j2 new file mode 100644 index 0000000..8830c45 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/configuration.xsl.j2 @@ -0,0 +1,42 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +<?xml version="1.0"?> +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"; version="1.0"> +<xsl:output method="html"/> +<xsl:template match="configuration"> +<html> +<body> +<table border="1"> +<tr> + <td>name</td> + <td>value</td> + <td>description</td> +</tr> +<xsl:for-each select="property"> + <tr> + <td><a name="{name}"><xsl:value-of select="name"/></a></td> + <td><xsl:value-of select="value"/></td> + <td><xsl:value-of select="description"/></td> + </tr> +</xsl:for-each> +</table> +</body> +</html> +</xsl:template> +</xsl:stylesheet> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zoo.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zoo.cfg.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zoo.cfg.j2 new file mode 100644 index 0000000..beb4730 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zoo.cfg.j2 @@ -0,0 +1,69 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# + +# The number of milliseconds of each tick +tickTime={{tickTime}} +# The number of ticks that the initial +# synchronization phase can take +initLimit={{initLimit}} +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit={{syncLimit}} +# the directory where the snapshot is stored. +dataDir={{zk_data_dir}} +# the port at which the clients will connect +clientPort={{clientPort}} +{% for host in zookeeper_hosts %} +server.{{loop.index}}={{host}}:2888:3888 +{% endfor %} + +{% if security_enabled %} +authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider +jaasLoginRenew=3600000 +kerberos.removeHostFromPrincipal=true +kerberos.removeRealmFromPrincipal=true +{% endif %} + +{% if zoo_cfg_properties_map_length > 0 %} +# Custom properties +{% endif %} +{% for key, value in zoo_cfg_properties_map.iteritems() %} +{{key}}={{value}} +{% endfor %} http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zookeeper_client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zookeeper_client_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zookeeper_client_jaas.conf.j2 new file mode 100644 index 0000000..38f9721 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zookeeper_client_jaas.conf.j2 @@ -0,0 +1,23 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=false +useTicketCache=true; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zookeeper_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zookeeper_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zookeeper_jaas.conf.j2 new file mode 100644 index 0000000..c3e9505 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/ZOOKEEPER/package/templates/zookeeper_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Server { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{zk_keytab_path}}" +principal="{{zk_principal}}"; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d07030/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/stack_advisor.py new file mode 100644 index 0000000..9052509 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/stack_advisor.py @@ -0,0 +1,443 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import re +import sys +from math import ceil + +from stack_advisor import DefaultStackAdvisor + +class BasePHD3000StackAdvisor(DefaultStackAdvisor): + + def getComponentLayoutValidations(self, services, hosts): + """Returns array of Validation objects about issues with hostnames components assigned to""" + items = [] + + # Validating NAMENODE and SECONDARY_NAMENODE are on different hosts if possible + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + hostsCount = len(hostsList) + + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + nameNodeHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "NAMENODE"] + secondaryNameNodeHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "SECONDARY_NAMENODE"] + + # Validating cardinality + for component in componentsList: + if component["StackServiceComponents"]["cardinality"] is not None: + componentName = component["StackServiceComponents"]["component_name"] + componentDisplayName = component["StackServiceComponents"]["display_name"] + componentHostsCount = 0 + if component["StackServiceComponents"]["hostnames"] is not None: + componentHostsCount = len(component["StackServiceComponents"]["hostnames"]) + cardinality = str(component["StackServiceComponents"]["cardinality"]) + # cardinality types: null, 1+, 1-2, 1, ALL + message = None + if "+" in cardinality: + hostsMin = int(cardinality[:-1]) + if componentHostsCount < hostsMin: + message = "At least {0} {1} components should be installed in cluster.".format(hostsMin, componentDisplayName) + elif "-" in cardinality: + nums = cardinality.split("-") + hostsMin = int(nums[0]) + hostsMax = int(nums[1]) + if componentHostsCount > hostsMax or componentHostsCount < hostsMin: + message = "Between {0} and {1} {2} components should be installed in cluster.".format(hostsMin, hostsMax, componentDisplayName) + elif "ALL" == cardinality: + if componentHostsCount != hostsCount: + message = "{0} component should be installed on all hosts in cluster.".format(componentDisplayName) + else: + if componentHostsCount != int(cardinality): + message = "Exactly {0} {1} components should be installed in cluster.".format(int(cardinality), componentDisplayName) + + if message is not None: + items.append({"type": 'host-component', "level": 'ERROR', "message": message, "component-name": componentName}) + + # Validating host-usage + usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)] + usedHostsList = [item for sublist in usedHostsListList for item in sublist] + nonUsedHostsList = [item for item in hostsList if item not in usedHostsList] + for host in nonUsedHostsList: + items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not used', "host": str(host) } ) + + return items + + def getServiceConfigurationRecommenderDict(self): + return { + "YARN": self.recommendYARNConfigurations, + "MAPREDUCE2": self.recommendMapReduce2Configurations + } + + def putProperty(self, config, configType): + config[configType] = {"properties": {}} + def appendProperty(key, value): + config[configType]["properties"][key] = str(value) + return appendProperty + + def recommendYARNConfigurations(self, configurations, clusterData): + putYarnProperty = self.putProperty(configurations, "yarn-site") + putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) + putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) + putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) + + def recommendMapReduce2Configurations(self, configurations, clusterData): + putMapredProperty = self.putProperty(configurations, "mapred-site") + putMapredProperty('yarn.app.mapreduce.am.resource.mb', int(clusterData['amMemory'])) + putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(round(0.8 * clusterData['amMemory']))) + "m") + putMapredProperty('mapreduce.map.memory.mb', clusterData['mapMemory']) + putMapredProperty('mapreduce.reduce.memory.mb', int(clusterData['reduceMemory'])) + putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m") + putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m") + putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024)) + + def getConfigurationClusterSummary(self, servicesList, hosts, components): + + hBaseInstalled = False + if 'HBASE' in servicesList: + hBaseInstalled = True + + cluster = { + "cpu": 0, + "disk": 0, + "ram": 0, + "hBaseInstalled": hBaseInstalled, + "components": components + } + + if len(hosts["items"]) > 0: + host = hosts["items"][0]["Hosts"] + cluster["cpu"] = host["cpu_count"] + cluster["disk"] = len(host["disk_info"]) + cluster["ram"] = int(host["total_mem"] / (1024 * 1024)) + + ramRecommendations = [ + {"os":1, "hbase":1}, + {"os":2, "hbase":1}, + {"os":2, "hbase":2}, + {"os":4, "hbase":4}, + {"os":6, "hbase":8}, + {"os":8, "hbase":8}, + {"os":8, "hbase":8}, + {"os":12, "hbase":16}, + {"os":24, "hbase":24}, + {"os":32, "hbase":32}, + {"os":64, "hbase":64} + ] + index = { + cluster["ram"] <= 4: 0, + 4 < cluster["ram"] <= 8: 1, + 8 < cluster["ram"] <= 16: 2, + 16 < cluster["ram"] <= 24: 3, + 24 < cluster["ram"] <= 48: 4, + 48 < cluster["ram"] <= 64: 5, + 64 < cluster["ram"] <= 72: 6, + 72 < cluster["ram"] <= 96: 7, + 96 < cluster["ram"] <= 128: 8, + 128 < cluster["ram"] <= 256: 9, + 256 < cluster["ram"]: 10 + }[1] + cluster["reservedRam"] = ramRecommendations[index]["os"] + cluster["hbaseRam"] = ramRecommendations[index]["hbase"] + + cluster["minContainerSize"] = { + cluster["ram"] <= 4: 256, + 4 < cluster["ram"] <= 8: 512, + 8 < cluster["ram"] <= 24: 1024, + 24 < cluster["ram"]: 2048 + }[1] + + totalAvailableRam = cluster["ram"] - cluster["reservedRam"] + if cluster["hBaseInstalled"]: + totalAvailableRam -= cluster["hbaseRam"] + cluster["totalAvailableRam"] = max(2048, totalAvailableRam * 1024) + '''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))''' + cluster["containers"] = round(max(3, + min(2 * cluster["cpu"], + min(ceil(1.8 * cluster["disk"]), + cluster["totalAvailableRam"] / cluster["minContainerSize"])))) + + '''ramPerContainers = max(2GB, RAM - reservedRam - hBaseRam) / containers''' + cluster["ramPerContainer"] = abs(cluster["totalAvailableRam"] / cluster["containers"]) + '''If greater than 1GB, value will be in multiples of 512.''' + if cluster["ramPerContainer"] > 1024: + cluster["ramPerContainer"] = int(cluster["ramPerContainer"] / 512) * 512 + + cluster["mapMemory"] = int(cluster["ramPerContainer"]) + cluster["reduceMemory"] = cluster["ramPerContainer"] + cluster["amMemory"] = max(cluster["mapMemory"], cluster["reduceMemory"]) + + return cluster + + def getConfigurationsValidationItems(self, services, hosts): + """Returns array of Validation objects about issues with configuration values provided in services""" + items = [] + + recommendations = self.recommendConfigurations(services, hosts) + recommendedDefaults = recommendations["recommendations"]["blueprint"]["configurations"] + + configurations = services["configurations"] + for service in services["services"]: + serviceName = service["StackServices"]["service_name"] + validator = self.validateServiceConfigurations(serviceName) + if validator is not None: + siteName = validator[0] + method = validator[1] + if siteName in recommendedDefaults: + siteProperties = getSiteProperties(configurations, siteName) + if siteProperties is not None: + resultItems = method(siteProperties, recommendedDefaults[siteName]["properties"], configurations) + items.extend(resultItems) + return items + + def getServiceConfigurationValidators(self): + return { + "MAPREDUCE2": ["mapred-site", self.validateMapReduce2Configurations], + "YARN": ["yarn-site", self.validateYARNConfigurations] + } + + def validateServiceConfigurations(self, serviceName): + return self.getServiceConfigurationValidators().get(serviceName, None) + + def toConfigurationValidationProblems(self, validationProblems, siteName): + result = [] + for validationProblem in validationProblems: + validationItem = validationProblem.get("item", None) + if validationItem is not None: + problem = {"type": 'configuration', "level": validationItem["level"], "message": validationItem["message"], + "config-type": siteName, "config-name": validationProblem["config-name"] } + result.append(problem) + return result + + def getWarnItem(self, message): + return {"level": "WARN", "message": message} + + def getErrorItem(self, message): + return {"level": "ERROR", "message": message} + + def validatorLessThenDefaultValue(self, properties, recommendedDefaults, propertyName): + if not propertyName in properties: + return self.getErrorItem("Value should be set") + value = to_number(properties[propertyName]) + if value is None: + return self.getErrorItem("Value should be integer") + defaultValue = to_number(recommendedDefaults[propertyName]) + if defaultValue is None: + return None + if value < defaultValue: + return self.getWarnItem("Value is less than the recommended default of {0}".format(defaultValue)) + return None + + def validateXmxValue(self, properties, recommendedDefaults, propertyName): + if not propertyName in properties: + return self.getErrorItem("Value should be set") + value = properties[propertyName] + defaultValue = recommendedDefaults[propertyName] + if defaultValue is None: + return self.getErrorItem("Config's default value can't be null or undefined") + if not checkXmxValueFormat(value): + return self.getErrorItem('Invalid value format') + valueInt = formatXmxSizeToBytes(getXmxSize(value)) + defaultValueXmx = getXmxSize(defaultValue) + defaultValueInt = formatXmxSizeToBytes(defaultValueXmx) + if valueInt < defaultValueInt: + return self.getWarnItem("Value is less than the recommended default of -Xmx" + defaultValueXmx) + return None + + def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations): + validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')}, + {"config-name": 'mapreduce.reduce.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')}, + {"config-name": 'mapreduce.task.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')}, + {"config-name": 'mapreduce.map.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')}, + {"config-name": 'mapreduce.reduce.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')}, + {"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')}, + {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ] + return self.toConfigurationValidationProblems(validationItems, "mapred-site") + + def validateYARNConfigurations(self, properties, recommendedDefaults, configurations): + validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')}, + {"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')}, + {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ] + return self.toConfigurationValidationProblems(validationItems, "yarn-site") + + def getMastersWithMultipleInstances(self): + return ['ZOOKEEPER_SERVER', 'HBASE_MASTER'] + + def getNotValuableComponents(self): + return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR'] + + def getNotPreferableOnServerComponents(self): + return ['GANGLIA_SERVER', 'NAGIOS_SERVER'] + + def getCardinalitiesDict(self): + return { + 'ZOOKEEPER_SERVER': {"min": 3}, + 'HBASE_MASTER': {"min": 1}, + } + + def getComponentLayoutSchemes(self): + return { + 'NAMENODE': {"else": 0}, + 'SECONDARY_NAMENODE': {"else": 1}, + 'HBASE_MASTER': {6: 0, 31: 2, "else": 3}, + + 'HISTORYSERVER': {31: 1, "else": 2}, + 'RESOURCEMANAGER': {31: 1, "else": 2}, + + 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3}, + + 'HIVE_SERVER': {6: 1, 31: 2, "else": 4}, + 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4}, + 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4}, + } + +class PHD3000StackAdvisor(BasePHD3000StackAdvisor): + + def getServiceConfigurationRecommenderDict(self): + parentRecommendConfDict = super(PHD3000StackAdvisor, self).getServiceConfigurationRecommenderDict() + childRecommendConfDict = { + "OOZIE": self.recommendOozieConfigurations, + "HIVE": self.recommendHiveConfigurations, + "TEZ": self.recommendTezConfigurations + } + parentRecommendConfDict.update(childRecommendConfDict) + return parentRecommendConfDict + + def recommendOozieConfigurations(self, configurations, clusterData): + if "FALCON_SERVER" in clusterData["components"]: + putMapredProperty = self.putProperty(configurations, "oozie-site") + putMapredProperty("oozie.services.ext", + "org.apache.oozie.service.JMSAccessorService," + + "org.apache.oozie.service.PartitionDependencyManagerService," + + "org.apache.oozie.service.HCatAccessorService") + + def recommendHiveConfigurations(self, configurations, clusterData): + containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory']) + containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize) + putHiveProperty = self.putProperty(configurations, "hive-site") + putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(round(containerSize / 3)) * 1048576) + putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(round(0.8 * containerSize))) + + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC") + putHiveProperty('hive.tez.container.size', containerSize) + + def recommendTezConfigurations(self, configurations, clusterData): + putTezProperty = self.putProperty(configurations, "tez-site") + putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory'])) + putTezProperty("tez.am.java.opts", + "-server -Xmx" + str(int(0.8 * clusterData["amMemory"])) + + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC") + + def getNotPreferableOnServerComponents(self): + return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER', 'NAGIOS_SERVER'] + + def getNotValuableComponents(self): + return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER'] + + def getComponentLayoutSchemes(self): + parentSchemes = super(PHD3000StackAdvisor, self).getComponentLayoutSchemes() + childSchemes = { + 'APP_TIMELINE_SERVER': {31: 1, "else": 2}, + 'FALCON_SERVER': {6: 1, 31: 2, "else": 3} + } + parentSchemes.update(childSchemes) + return parentSchemes + + def getServiceConfigurationValidators(self): + parentValidators = super(PHD3000StackAdvisor, self).getServiceConfigurationValidators() + childValidators = { + "HIVE": ["hive-site", self.validateHiveConfigurations], + "TEZ": ["tez-site", self.validateTezConfigurations] + } + parentValidators.update(childValidators) + return parentValidators + + def validateHiveConfigurations(self, properties, recommendedDefaults, configurations): + validationItems = [ {"config-name": 'hive.tez.container.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')}, + {"config-name": 'hive.tez.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')}, + {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ] + return self.toConfigurationValidationProblems(validationItems, "hive-site") + + def validateTezConfigurations(self, properties, recommendedDefaults, configurations): + validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')}, + {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ] + return self.toConfigurationValidationProblems(validationItems, "tez-site") + +# Validation helper methods +def getSiteProperties(configurations, siteName): + siteConfig = configurations.get(siteName) + if siteConfig is None: + return None + return siteConfig.get("properties") + +def to_number(s): + try: + return int(re.sub("\D", "", s)) + except ValueError: + return None + +def checkXmxValueFormat(value): + p = re.compile('-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)?') + matches = p.findall(value) + return len(matches) == 1 + +def getXmxSize(value): + p = re.compile("-Xmx(\d+)(.?)") + result = p.findall(value)[0] + if len(result) > 1: + # result[1] - is a space or size formatter (b|k|m|g etc) + return result[0] + result[1].lower() + return result[0] + +def formatXmxSizeToBytes(value): + value = value.lower() + if len(value) == 0: + return 0 + modifier = value[-1] + + if modifier == ' ' or modifier in "0123456789": + modifier = 'b' + m = { + modifier == 'b': 1, + modifier == 'k': 1024, + modifier == 'm': 1024 * 1024, + modifier == 'g': 1024 * 1024 * 1024, + modifier == 't': 1024 * 1024 * 1024 * 1024, + modifier == 'p': 1024 * 1024 * 1024 * 1024 * 1024 + }[1] + return to_number(value) * m + +def getPort(address): + """ + Extracts port from the address like 0.0.0.0:1019 + """ + if address is None: + return None + m = re.search(r'(?:http(?:s)?://)?([\w\d.]*):(\d{1,5})', address) + if m is not None: + return int(m.group(2)) + else: + return None + +def isSecurePort(port): + """ + Returns True if port is root-owned at *nix systems + """ + if port is not None: + return port < 1024 + else: + return False
