Repository: ambari Updated Branches: refs/heads/branch-metrics-dev 5649e80bb -> 1200d7288
http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-site.xml new file mode 100644 index 0000000..72ea95f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-site.xml @@ -0,0 +1,221 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>timeline.metrics.service.operation.mode</name> + <value>embedded</value> + <description> + Service Operation modes: + 1) embedded: Metrics stored on local FS, HBase in Standalone mode + 2) distributed: HBase daemons writing to HDFS + 3) external: External HBase storage backend + </description> + </property> + <property> + <name>timeline.metrics.aggregator.checkpoint.dir</name> + <value>/tmp</value> + <description> + Directory to store aggregator checkpoints. Change to a permanent + location so that checkpoint ar not lost. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.hourly.interval</name> + <value>3600</value> + <description> + Time in seconds to sleep for the hourly resolution host based + aggregator. Default resolution is 1 hour. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.minute.interval</name> + <value>300</value> + <description> + Time in seconds to sleep for the minute resolution host based + aggregator. Default resolution is 5 minutes. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.hourly.interval</name> + <value>3600</value> + <description> + Time in seconds to sleep for the hourly resolution cluster wide + aggregator. Default is 1 hour. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.minute.interval</name> + <value>120</value> + <description> + Time in seconds to sleep for the minute resolution cluster wide + aggregator. Default resolution is 2 minutes. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.hourly.checkpointCutOffMultiplier</name> + <value>2</value> + <description> + Multiplier value * interval = Max allowed checkpoint lag. Effectively + if aggregator checkpoint is greater than max allowed checkpoint delay, + the checkpoint will be discarded by the aggregator. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.minute.checkpointCutOffMultiplier</name> + <value>2</value> + <description> + Multiplier value * interval = Max allowed checkpoint lag. Effectively + if aggregator checkpoint is greater than max allowed checkpoint delay, + the checkpoint will be discarded by the aggregator. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.hourly.checkpointCutOffMultiplier</name> + <value>2</value> + <description> + Multiplier value * interval = Max allowed checkpoint lag. Effectively + if aggregator checkpoint is greater than max allowed checkpoint delay, + the checkpoint will be discarded by the aggregator. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.minute.checkpointCutOffMultiplier</name> + <value>2</value> + <description> + Multiplier value * interval = Max allowed checkpoint lag. Effectively + if aggregator checkpoint is greater than max allowed checkpoint delay, + the checkpoint will be discarded by the aggregator. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.hourly.disabled</name> + <value>false</value> + <description> + Disable host based hourly aggregations. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.minute.disabled</name> + <value>false</value> + <description> + Disable host based minute aggregations. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.hourly.disabled</name> + <value>false</value> + <description> + Disable cluster based hourly aggregations. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.minute.disabled</name> + <value>false</value> + <description> + Disable cluster based minute aggregations. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.minute.timeslice.interval</name> + <value>15</value> + <description> + Lowest resolution of desired data for cluster level minute aggregates. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.hourly.ttl</name> + <value>2592000</value> + <description> + Host based hourly resolution data purge interval. Default is 30 days. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.minute.ttl</name> + <value>604800</value> + <description> + Host based minute resolution data purge interval. Default is 7 days. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.minute.ttl</name> + <value>2592000</value> + <description> + Cluster wide minute resolution data purge interval. Default is 30 days. + </description> + </property> + <property> + <name>timeline.metrics.cluster.aggregator.hourly.ttl</name> + <value>31536000</value> + <description> + Cluster wide hourly resolution data purge interval. Default is 1 year. + </description> + </property> + <property> + <name>timeline.metrics.host.aggregator.ttl</name> + <value>86400</value> + <description> + 1 minute resolution data purge interval. Default is 1 day. + </description> + </property> + <property> + <name>timeline.metrics.hbase.data.block.encoding</name> + <value>FAST_DIFF</value> + <description> + Codecs are enabled on a table by setting the DATA_BLOCK_ENCODING property. + Default encoding is FAST_DIFF. This can be changed only before creating + tables. + </description> + </property> + <property> + <name>timeline.metrics.hbase.compression.scheme</name> + <value>SNAPPY</value> + <description> + Compression codes need to be installed and available before setting the + scheme. Default compression is SNAPPY. Disable by setting to None. + This can be changed only before creating tables. + </description> + </property> + <property> + <name>timeline.metrics.service.default.result.limit</name> + <value>5760</value> + <description> + Max result limit on number of rows returned. Calculated as follows: + 4 aggregate metrics/min * 60 * 24: Retrieve aggregate data for 1 day. + </description> + </property> + <property> + <name>timeline.metrics.service.checkpointDelay</name> + <value>120000</value> + <description> + Time in milliseconds to sleep on the first run or when the checkpoint is + too old. + </description> + </property> + <property> + <name>timeline.metrics.service.resultset.fetchSize</name> + <value>2000</value> + <description> + JDBC resultset prefect size for aggregator queries. + </description> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/metainfo.xml new file mode 100644 index 0000000..537f2e6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/metainfo.xml @@ -0,0 +1,97 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>AMS</name> + <displayName>AMS</displayName> + <version>0.1.0</version> + <comment>Ambari Metrics Service - Monitoring service that provides + storage and retrieval capability for metrics collected from services + & hosts. + </comment> + <components> + <component> + <name>METRIC_COLLECTOR</name> + <displayName>Metric Collector</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <dependencies> + <dependency> + <name>ZOOKEEPER/ZOOKEEPER_SERVER</name> + <scope>cluster</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/metric_collector.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>METRIC_MONITOR</name> + <displayName>Metric monitor</displayName> + <category>SLAVE</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/metric_monitor.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>ambari-metrics-collector</name> + </package> + <package> + <name>ambari-metrics-monitor</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>ZOOKEEPER</service> + </requiredServices> + + <configuration-dependencies> + <config-type>ams-site</config-type> + <config-type>hbase-ams-policy</config-type> + <config-type>hbase-ams-site</config-type> + <config-type>hbase-ams-env</config-type> + <config-type>hbase-ams-log4j</config-type> + </configuration-dependencies> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/files/hbaseSmokeVerify.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/files/hbaseSmokeVerify.sh b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/files/hbaseSmokeVerify.sh new file mode 100644 index 0000000..5c320c0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/files/hbaseSmokeVerify.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +conf_dir=$1 +data=$2 +hbase_cmd=$3 +echo "scan 'ambarismoketest'" | $hbase_cmd --config $conf_dir shell > /tmp/hbase_chk_verify +cat /tmp/hbase_chk_verify +echo "Looking for $data" +grep -q $data /tmp/hbase_chk_verify +if [ "$?" -ne 0 ] +then + exit 1 +fi + +grep -q '1 row(s)' /tmp/hbase_chk_verify http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/__init__.py new file mode 100644 index 0000000..5561e10 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase.py new file mode 100644 index 0000000..2829c7b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os + +from resource_management import * +import sys + +def hbase(name=None # 'master' or 'regionserver' or 'client' + ): + import params + + Directory( params.hbase_conf_dir, + owner = params.hbase_user, + group = params.user_group, + recursive = True + ) + + Directory (params.tmp_dir, + owner = params.hbase_user, + recursive = True + ) + + Directory (os.path.join(params.local_dir, "jars"), + owner = params.hbase_user, + group = params.user_group, + mode=0775, + recursive = True + ) + + XmlConfig( "hbase-site.xml", + conf_dir = params.hbase_conf_dir, + configurations = params.config['configurations']['hbase-site'], + configuration_attributes=params.config['configuration_attributes']['hbase-site'], + owner = params.hbase_user, + group = params.user_group + ) + + XmlConfig( "hdfs-site.xml", + conf_dir = params.hbase_conf_dir, + configurations = params.config['configurations']['hdfs-site'], + configuration_attributes=params.config['configuration_attributes']['hdfs-site'], + owner = params.hbase_user, + group = params.user_group + ) + + XmlConfig("hdfs-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['hdfs-site'], + configuration_attributes=params.config['configuration_attributes']['hdfs-site'], + owner=params.hdfs_user, + group=params.user_group + ) + + if 'hbase-policy' in params.config['configurations']: + XmlConfig( "hbase-policy.xml", + conf_dir = params.hbase_conf_dir, + configurations = params.config['configurations']['hbase-policy'], + configuration_attributes=params.config['configuration_attributes']['hbase-policy'], + owner = params.hbase_user, + group = params.user_group + ) + # Manually overriding ownership of file installed by hadoop package + else: + File( format("{params.hbase_conf_dir}/hbase-policy.xml"), + owner = params.hbase_user, + group = params.user_group + ) + + File(format("{hbase_conf_dir}/hbase-env.sh"), + owner = params.hbase_user, + content=InlineTemplate(params.hbase_env_sh_template) + ) + + hbase_TemplateConfig( params.metric_prop_file_name, + tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS' + ) + + hbase_TemplateConfig( 'regionservers') + + if params.security_enabled: + hbase_TemplateConfig( format("hbase_{name}_jaas.conf")) + + if name != "client": + Directory( params.pid_dir, + owner = params.hbase_user, + recursive = True + ) + + Directory (params.log_dir, + owner = params.hbase_user, + recursive = True + ) + + if (params.log4j_props != None): + File(format("{params.hbase_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hbase_user, + content=params.log4j_props + ) + elif (os.path.exists(format("{params.hbase_conf_dir}/log4j.properties"))): + File(format("{params.hbase_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hbase_user + ) + if name in ["master","regionserver"]: + params.HdfsDirectory(params.hbase_hdfs_root_dir, + action="create_delayed", + owner=params.hbase_user + ) + params.HdfsDirectory(params.hbase_staging_dir, + action="create_delayed", + owner=params.hbase_user, + mode=0711 + ) + params.HdfsDirectory(None, action="create") + +def hbase_TemplateConfig(name, + tag=None + ): + import params + + TemplateConfig( format("{hbase_conf_dir}/{name}"), + owner = params.hbase_user, + template_tag = tag + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_master.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_master.py new file mode 100644 index 0000000..a26254d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_master.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service +from hbase_decommission import hbase_decommission + + +class HbaseMaster(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='master') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + hbase_service( 'master', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'master', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid") + check_process_status(pid_file) + + def decommission(self, env): + import params + env.set_params(params) + + hbase_decommission(env) + + +if __name__ == "__main__": + HbaseMaster().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_regionserver.py new file mode 100644 index 0000000..8d66dcc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_regionserver.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service + + +class HbaseRegionServer(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='regionserver') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + hbase_service( 'regionserver', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'regionserver', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid") + check_process_status(pid_file) + + def decommission(self, env): + print "Decommission not yet implemented!" + + +if __name__ == "__main__": + HbaseRegionServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_service.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_service.py new file mode 100644 index 0000000..723d4e2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/hbase_service.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +def hbase_service( + name, + action = 'start'): # 'start' or 'stop' or 'status' + + import params + + role = name + cmd = format("{daemon_script} --config {hbase_conf_dir}") + pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid") + no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + + if action == 'start': + daemon_cmd = format("{cmd} start {role}") + + Execute ( daemon_cmd, + not_if = no_op_test, + user = params.hbase_user + ) + elif action == 'stop': + daemon_cmd = format("{cmd} stop {role}") + + Execute ( daemon_cmd, + user = params.hbase_user, + # BUGFIX: hbase regionserver sometimes hangs when nn is in safemode + timeout = 30, + on_timeout = format("{no_op_test} && kill -9 `cat {pid_file}`") + ) + + Execute (format("rm -f {pid_file}")) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py new file mode 100644 index 0000000..c31caf2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from functions import calc_xmn_from_xms +from resource_management import * +import status_params + +# server configurations +config = Script.get_config() +exec_tmp_dir = Script.get_tmp_dir() + +#RPM versioning support +rpm_version = default("/configurations/hadoop-env/rpm_version", None) + +#hadoop params +if rpm_version is not None: +#RPM versioning support + rpm_version = default("/configurations/hadoop-env/rpm_version", None) + +#hadoop params +if rpm_version is not None: + hadoop_bin_dir = format("/usr/hdp/current/hadoop/bin") + daemon_script = format('/usr/hdp/current/hbase/bin/hbase-daemon.sh') + region_mover = format('/usr/hdp/current/hbase/bin/region_mover.rb') + region_drainer = format('/usr/hdp/currenthbase/bin/draining_servers.rb') + hbase_cmd = format('/usr/hdp/current/hbase/bin/hbase') +else: + hadoop_bin_dir = "/usr/bin" + daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh" + region_mover = "/usr/lib/hbase/bin/region_mover.rb" + region_drainer = "/usr/lib/hbase/bin/draining_servers.rb" + hbase_cmd = "/usr/lib/hbase/bin/hbase" + +hadoop_conf_dir = "/etc/hadoop/conf" +hbase_conf_dir = "/etc/hbase/conf" +hbase_excluded_hosts = config['commandParams']['excluded_hosts'] +hbase_drain_only = config['commandParams']['mark_draining_only'] +hbase_included_hosts = config['commandParams']['included_hosts'] + +hbase_user = status_params.hbase_user +smokeuser = config['configurations']['cluster-env']['smokeuser'] +_authentication = config['configurations']['core-site']['hadoop.security.authentication'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] + +# this is "hadoop-metrics.properties" for 1.x stacks +metric_prop_file_name = "hadoop-metrics2-hbase.properties" + +# not supporting 32 bit jdk. +java64_home = config['hostLevelParams']['java_home'] + +log_dir = config['configurations']['hbase-env']['hbase_log_dir'] +master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize'] + +regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize'] +regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max'] +regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio'] +regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) + +pid_dir = status_params.pid_dir +tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] +# TODO UPGRADE default, update site during upgrade +_local_dir_conf = default('/configurations/hbase-site/hbase.local.dir', "${hbase.tmp.dir}/local") +local_dir = substitute_vars(_local_dir_conf, config['configurations']['hbase-site']) + +client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf") +master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf") +regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf") + +ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present +ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0] + +# if hbase is selected the hbase_rs_hosts, should not be empty, but still default just in case +if 'slave_hosts' in config['clusterHostInfo']: + rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/slave_hosts') #if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves +else: + rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/all_hosts') + +smoke_test_user = config['configurations']['cluster-env']['smokeuser'] +smokeuser_permissions = "RWXCA" +service_check_data = functions.get_unique_id_and_date() +user_group = config['configurations']['cluster-env']["user_group"] + +if security_enabled: + _hostname_lowercase = config['hostname'].lower() + master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase) + regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase) + +master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] +regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab'] +kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +if security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};") +else: + kinit_cmd = "" + +#log4j.properties +if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])): + log4j_props = config['configurations']['hbase-log4j']['content'] +else: + log4j_props = None + +hbase_env_sh_template = config['configurations']['hbase-env']['content'] + +hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir'] +hbase_staging_dir = "/apps/hbase/staging" +#for create_hdfs_directory +hostname = config["hostname"] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +import functools +#create partial functions with common arguments for every HdfsDirectory call +#to create hdfs directory we need to call params.HdfsDirectory in code +HdfsDirectory = functools.partial( + HdfsDirectory, + conf_dir=hadoop_conf_dir, + hdfs_user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + bin_dir = hadoop_bin_dir +) http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_client_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_client_jaas.conf.j2 new file mode 100644 index 0000000..38f9721 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_client_jaas.conf.j2 @@ -0,0 +1,23 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=false +useTicketCache=true; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_grant_permissions.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_grant_permissions.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_grant_permissions.j2 new file mode 100644 index 0000000..3378983 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_grant_permissions.j2 @@ -0,0 +1,39 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +grant '{{smoke_test_user}}', '{{smokeuser_permissions}}' +exit \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_master_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_master_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_master_jaas.conf.j2 new file mode 100644 index 0000000..a93c36c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_master_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{master_keytab_path}}" +principal="{{master_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_regionserver_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_regionserver_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_regionserver_jaas.conf.j2 new file mode 100644 index 0000000..7097481 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hbase_regionserver_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{regionserver_keytab_path}}" +principal="{{regionserver_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/78dbf4f8/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/regionservers.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/regionservers.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/regionservers.j2 new file mode 100644 index 0000000..fc6cc37 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/regionservers.j2 @@ -0,0 +1,20 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in rs_hosts %}{{host}} +{% endfor %} \ No newline at end of file
