SLIDER-288 Add role for HBase thrift2 gateway
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6f912df2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6f912df2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6f912df2 Branch: refs/heads/feature/SLIDER-151_REST_API Commit: 6f912df214136a8cd903b0c5d17ee9d680e6cffc Parents: a1bbf4d Author: tedyu <yuzhih...@gmail.com> Authored: Fri Aug 8 11:24:28 2014 -0700 Committer: tedyu <yuzhih...@gmail.com> Committed: Fri Aug 8 11:24:28 2014 -0700 ---------------------------------------------------------------------- app-packages/hbase/appConfig.json | 1 + app-packages/hbase/links.py | 20 +++++++ app-packages/hbase/metainfo.xml | 15 +++++ .../hbase/package/scripts/hbase_service.py | 2 + .../hbase/package/scripts/hbase_thrift2.py | 62 ++++++++++++++++++++ app-packages/hbase/package/scripts/params.py | 1 + app-packages/hbase/resources.json | 9 ++- .../providers/hbase/HBaseClientProvider.java | 4 ++ .../slider/providers/hbase/HBaseKeys.java | 5 ++ .../providers/hbase/HBaseProviderService.java | 5 ++ .../slider/providers/hbase/HBaseRoles.java | 2 + 11 files changed, 124 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json index f0ab144..24d9a84 100644 --- a/app-packages/hbase/appConfig.json +++ b/app-packages/hbase/appConfig.json @@ -24,6 +24,7 @@ "site.global.ganglia_server_port": "8667", "site.global.ganglia_server_id": "Application1", "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}", + "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}", "site.hbase-site.hbase.hstore.flush.retries.number": "120", "site.hbase-site.hbase.client.keyvalue.maxsize": "10485760", "site.hbase-site.hbase.hstore.compactionThreshold": "3", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/links.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase/links.py b/app-packages/hbase/links.py index c77e620..19b0d91 100644 --- a/app-packages/hbase/links.py +++ b/app-packages/hbase/links.py @@ -1,3 +1,23 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + from __future__ import print_function import logging import json http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml index ee6ddea..a5dab0c 100644 --- a/app-packages/hbase/metainfo.xml +++ b/app-packages/hbase/metainfo.xml @@ -45,6 +45,10 @@ <value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value> </export> <export> + <name>org.apache.slider.hbase.thrift2</name> + <value>http://${HBASE_THRIFT2_HOST}:${site.global.hbase_thrift2_port}</value> + </export> + <export> <name>org.apache.slider.hbase.thrift</name> <value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value> </export> @@ -68,6 +72,7 @@ <command>HBASE_MASTER-START</command> <requires>HBASE_REST-INSTALLED</requires> <requires>HBASE_THRIFT-INSTALLED</requires> + <requires>HBASE_THRIFT2-INSTALLED</requires> </commandOrder> </commandOrders> <components> @@ -114,6 +119,16 @@ </component> <component> + <name>HBASE_THRIFT2</name> + <category>SLAVE</category> + <minInstanceCount>1</minInstanceCount> + <commandScript> + <script>scripts/hbase_thrift2.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> + + <component> <name>HBASE_CLIENT</name> <category>CLIENT</category> <minInstanceCount>0</minInstanceCount> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/hbase_service.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase/package/scripts/hbase_service.py b/app-packages/hbase/package/scripts/hbase_service.py index 01d38b3..96add84 100644 --- a/app-packages/hbase/package/scripts/hbase_service.py +++ b/app-packages/hbase/package/scripts/hbase_service.py @@ -39,6 +39,8 @@ def hbase_service( daemon_cmd = format("{daemon_cmd} -p {rest_port}") elif name == 'thrift': daemon_cmd = format("{daemon_cmd} -p {thrift_port}") + elif name == 'thrift2': + daemon_cmd = format("{daemon_cmd} -p {thrift2_port}") no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") elif action == 'stop': daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/hbase_thrift2.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase/package/scripts/hbase_thrift2.py b/app-packages/hbase/package/scripts/hbase_thrift2.py new file mode 100644 index 0000000..b72196c --- /dev/null +++ b/app-packages/hbase/package/scripts/hbase_thrift2.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service + + +class HbaseThrift2(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='thrift2') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + hbase_service( 'thrift2', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'thrift2', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift2.pid") + check_process_status(pid_file) + +if __name__ == "__main__": + HbaseThrift2().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py index 2d05289..1f25f68 100644 --- a/app-packages/hbase/package/scripts/params.py +++ b/app-packages/hbase/package/scripts/params.py @@ -59,6 +59,7 @@ ganglia_server_port = default('/configurations/global/ganglia_server_port', '866 rest_port = config['configurations']['global']['hbase_rest_port'] thrift_port = config['configurations']['global']['hbase_thrift_port'] +thrift2_port = config['configurations']['global']['hbase_thrift2_port'] if security_enabled: http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/resources.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/resources.json b/app-packages/hbase/resources.json index 2d2b4f9..d2fdbd8 100644 --- a/app-packages/hbase/resources.json +++ b/app-packages/hbase/resources.json @@ -21,9 +21,14 @@ "yarn.role.priority": "3", "yarn.component.instances": "1", "yarn.memory": "256" - } + }, "HBASE_THRIFT": { - "yarn.role.priority": "3", + "yarn.role.priority": "4", + "yarn.component.instances": "1", + "yarn.memory": "256" + }, + "HBASE_THRIFT2": { + "yarn.role.priority": "5", "yarn.component.instances": "1", "yarn.memory": "256" } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java index fde2bf7..cd104a2 100644 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java +++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java @@ -239,8 +239,12 @@ public class HBaseClientProvider extends AbstractClientProvider implements providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_MASTER, 0, -1); + providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_REST_GATEWAY, + 0, -1); providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT_GATEWAY, 0, -1); + providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT2_GATEWAY, + 0, -1); } @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java index 9c38fb2..2a20438 100644 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java +++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java @@ -30,6 +30,8 @@ public interface HBaseKeys { String ROLE_THRIFT_GATEWAY = "THRIFT"; + String ROLE_THRIFT2_GATEWAY = "THRIFT2"; + /** {@value */ String REGION_SERVER = "regionserver"; @@ -38,6 +40,9 @@ public interface HBaseKeys { /** {@value */ String THRIFT_GATEWAY = "thrift"; + /** {@value */ + String THRIFT2_GATEWAY = "thrift2"; + /** /** * What is the command for hbase to print a version: {@value} */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java index 953ee79..f72967f 100644 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java +++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java @@ -201,6 +201,11 @@ public class HBaseProviderService extends AbstractProviderService implements logfile = "/thrift-gateway.txt"; break; + case ROLE_THRIFT2_GATEWAY: + roleCommand = THRIFT2_GATEWAY; + + logfile = "/thrift2-gateway.txt"; + break; default: throw new SliderInternalStateException("Cannot start role %s", role); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java index 2cc457b..b2825b8 100644 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java +++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java @@ -41,6 +41,7 @@ public class HBaseRoles { public static final int KEY_THRIFT_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 4; + public static final int KEY_THRIFT2_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 5; /** * Initialize role list */ @@ -49,6 +50,7 @@ public class HBaseRoles { ROLES.add(new ProviderRole(HBaseKeys.ROLE_MASTER, KEY_MASTER)); ROLES.add(new ProviderRole(HBaseKeys.ROLE_REST_GATEWAY, KEY_REST_GATEWAY)); ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT_GATEWAY)); + ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT2_GATEWAY)); }
