SLIDER-359. Create hbase package for Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/829821a0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/829821a0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/829821a0 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 829821a000cd175611ee346d1f48dae462469ea4 Parents: 7bc331d Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Mon Aug 25 13:54:40 2014 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Mon Aug 25 13:54:40 2014 -0700 ---------------------------------------------------------------------- app-packages/hbase-win/README.txt | 33 ++ app-packages/hbase-win/appConfig.json | 44 +++ .../hbase-win/configuration/hbase-env.xml | 117 ++++++ .../hbase-win/configuration/hbase-log4j.xml | 143 +++++++ .../hbase-win/configuration/hbase-policy.xml | 53 +++ .../hbase-win/configuration/hbase-site.xml | 370 +++++++++++++++++++ app-packages/hbase-win/jmx_metrics.json | 56 +++ app-packages/hbase-win/metainfo.xml | 119 ++++++ ...se-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME | 17 + .../hbase-win/package/scripts/__init__.py | 19 + .../hbase-win/package/scripts/functions.py | 40 ++ app-packages/hbase-win/package/scripts/hbase.py | 79 ++++ .../hbase-win/package/scripts/hbase_master.py | 63 ++++ .../package/scripts/hbase_regionserver.py | 63 ++++ .../hbase-win/package/scripts/hbase_service.py | 51 +++ .../hbase-win/package/scripts/params.py | 107 ++++++ .../hbase-win/package/scripts/status_params.py | 26 ++ .../package/templates/hbase_client_jaas.conf.j2 | 22 ++ .../package/templates/hbase_master_jaas.conf.j2 | 25 ++ .../templates/hbase_regionserver_jaas.conf.j2 | 25 ++ app-packages/hbase-win/resources.json | 21 ++ .../core/providers/windows/system.py | 79 +++- .../providers/agent/AgentProviderService.java | 2 + 23 files changed, 1560 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/README.txt ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt new file mode 100644 index 0000000..c122d90 --- /dev/null +++ b/app-packages/hbase-win/README.txt @@ -0,0 +1,33 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +Create Slider App Package for HBase on Windows + +Download the tarball for HBase: + e.g. path to tarball ~/Downloads/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip + +Copy the hbase tarball to package/files + cp ~/Downloads/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip package/files + +Edit appConfig.json/metainfo.xml and replace all occurances of +0.98.0.2.1.5.0-2047-hadoop2 with the correct value. + +Create a zip package at the root of the package (<slider enlistment>/app-packages/hbase-win/) + zip -r hbase-win-v098.zip . + +Verify the content using + zip -Tv hbase-win-v098.zip http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/appConfig.json b/app-packages/hbase-win/appConfig.json new file mode 100644 index 0000000..52f3aa3 --- /dev/null +++ b/app-packages/hbase-win/appConfig.json @@ -0,0 +1,44 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { + "application.def": "${app.package.name}.zip", + "create.default.zookeeper.node": "true", + "java_home": "/usr/jdk64/jdk1.7.0_45", + + "site.global.app_user": "hadoop", + "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}", + + "site.global.hbase_instance_name": "instancename", + "site.global.hbase_root_password": "secret", + "site.global.user_group": "hadoop", + "site.global.security_enabled": "false", + "site.global.monitor_protocol": "http", + "site.global.ganglia_server_host": "${NN_HOST}", + "site.global.ganglia_server_port": "8667", + "site.global.ganglia_server_id": "Application1", + "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}", + "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}", + "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}", + + "site.hbase-env.hbase_master_heapsize": "1024m", + "site.hbase-env.hbase_regionserver_heapsize": "1024m", + + "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data", + "site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging", + "site.hbase-site.hbase.superuser": "yarn", + "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp", + "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local", + "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}", + "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}", + "site.hbase-site.hbase.regionserver.info.port": "0", + "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}", + "site.hbase-site.hbase.regionserver.port": "0" + }, + "components": { + "slider-appmaster": { + "jvm.heapsize": "256M" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/configuration/hbase-env.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/configuration/hbase-env.xml b/app-packages/hbase-win/configuration/hbase-env.xml new file mode 100644 index 0000000..554c3c5 --- /dev/null +++ b/app-packages/hbase-win/configuration/hbase-env.xml @@ -0,0 +1,117 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>hbase_regionserver_heapsize</name> + <value>1024</value> + <description>HBase RegionServer Heap Size.</description> + </property> + <property> + <name>hbase_regionserver_xmn_max</name> + <value>512</value> + <description>HBase RegionServer maximum value for minimum heap size.</description> + </property> + <property> + <name>hbase_regionserver_xmn_ratio</name> + <value>0.2</value> + <description>HBase RegionServer minimum heap size is calculated as a percentage of max heap size.</description> + </property> + <property> + <name>hbase_master_heapsize</name> + <value>1024</value> + <description>HBase Master Heap Size</description> + </property> + + <!-- hbase-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for hbase-env.sh file</description> + <value> +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME={{java64_home}} + +# HBase Configuration directory +export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{conf_dir}}} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log" +export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}" +export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}" +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR={{log_dir}} + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR={{pid_dir}} + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false + +{% if security_enabled %} +export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}" +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}" +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}" +{% endif %} + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/configuration/hbase-log4j.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/configuration/hbase-log4j.xml b/app-packages/hbase-win/configuration/hbase-log4j.xml new file mode 100644 index 0000000..d488c4e --- /dev/null +++ b/app-packages/hbase-win/configuration/hbase-log4j.xml @@ -0,0 +1,143 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + + <property> + <name>content</name> + <description>Custom log4j.properties</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.security.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Rolling File Appender properties +hbase.log.maxfilesize=256MB +hbase.log.maxbackupindex=20 + +# Rolling File Appender +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file} + +log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize} +log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex} + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +hbase.security.log.maxfilesize=256MB +hbase.security.log.maxbackupindex=20 +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false +#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=INFO +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=DEBUG +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE + + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of .META. messages +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO +# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO + + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/configuration/hbase-policy.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/configuration/hbase-policy.xml b/app-packages/hbase-win/configuration/hbase-policy.xml new file mode 100644 index 0000000..e45f23c --- /dev/null +++ b/app-packages/hbase-win/configuration/hbase-policy.xml @@ -0,0 +1,53 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>security.client.protocol.acl</name> + <value>*</value> + <description>ACL for HRegionInterface protocol implementations (ie. + clients talking to HRegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + </property> + + <property> + <name>security.admin.protocol.acl</name> + <value>*</value> + <description>ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + </property> + + <property> + <name>security.masterregion.protocol.acl</name> + <value>*</value> + <description>ACL for HMasterRegionInterface protocol implementations + (for HRegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/configuration/hbase-site.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/configuration/hbase-site.xml b/app-packages/hbase-win/configuration/hbase-site.xml new file mode 100644 index 0000000..a9711d3 --- /dev/null +++ b/app-packages/hbase-win/configuration/hbase-site.xml @@ -0,0 +1,370 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>hbase.rootdir</name> + <value>hdfs://localhost:8020/apps/hbase/data</value> + <description>The directory shared by region servers and into + which HBase persists. The URL should be 'fully-qualified' + to include the filesystem scheme. For example, to specify the + HDFS directory '/hbase' where the HDFS instance's namenode is + running at namenode.example.org on port 9000, set this value to: + hdfs://namenode.example.org:9000/hbase. By default HBase writes + into /tmp. Change this configuration else all data will be lost + on machine restart. + </description> + </property> + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + <description>The mode the cluster will be in. Possible values are + false for standalone mode and true for distributed mode. If + false, startup will run all HBase and ZooKeeper daemons together + in the one JVM. + </description> + </property> + <property> + <name>hbase.master.port</name> + <value>60000</value> + <description>The port the HBase Master should bind to.</description> + </property> + <property> + <name>hbase.tmp.dir</name> + <value>/hadoop/hbase</value> + <description>Temporary directory on the local filesystem. + Change this setting to point to a location more permanent + than '/tmp' (The '/tmp' directory is often cleared on + machine restart). + </description> + </property> + <property> + <name>hbase.local.dir</name> + <value>${hbase.tmp.dir}/local</value> + <description>Directory on the local filesystem to be used as a local storage + </description> + </property> + <property> + <name>hbase.master.info.bindAddress</name> + <value>0.0.0.0</value> + <description>The bind address for the HBase Master web UI + </description> + </property> + <property> + <name>hbase.master.info.port</name> + <value>60010</value> + <description>The port for the HBase Master web UI.</description> + </property> + <property> + <name>hbase.regionserver.info.port</name> + <value>60030</value> + <description>The port for the HBase RegionServer web UI.</description> + </property> + <property> + <name>hbase.regionserver.global.memstore.upperLimit</name> + <value>0.4</value> + <description>Maximum size of all memstores in a region server before new + updates are blocked and flushes are forced. Defaults to 40% of heap + </description> + </property> + <property> + <name>hbase.regionserver.handler.count</name> + <value>60</value> + <description>Count of RPC Listener instances spun up on RegionServers. + Same property is used by the Master for count of master handlers. + Default is 10. + </description> + </property> + <property> + <name>hbase.hregion.majorcompaction</name> + <value>86400000</value> + <description>The time (in milliseconds) between 'major' compactions of all + HStoreFiles in a region. Default: 1 day. + Set to 0 to disable automated major compactions. + </description> + </property> + + <property> + <name>hbase.regionserver.global.memstore.lowerLimit</name> + <value>0.38</value> + <description>When memstores are being forced to flush to make room in + memory, keep flushing until we hit this mark. Defaults to 35% of heap. + This value equal to hbase.regionserver.global.memstore.upperLimit causes + the minimum possible flushing to occur when updates are blocked due to + memstore limiting. + </description> + </property> + <property> + <name>hbase.hregion.memstore.block.multiplier</name> + <value>2</value> + <description>Block updates if memstore has hbase.hregion.memstore.block.multiplier + time hbase.hregion.flush.size bytes. Useful preventing + runaway memstore during spikes in update traffic. Without an + upper-bound, memstore fills such that when it flushes the + resultant flush files take a long time to compact or split, or + worse, we OOME + </description> + </property> + <property> + <name>hbase.hregion.memstore.flush.size</name> + <value>134217728</value> + <description> + Memstore will be flushed to disk if size of the memstore + exceeds this number of bytes. Value is checked by a thread that runs + every hbase.server.thread.wakefrequency. + </description> + </property> + <property> + <name>hbase.hregion.memstore.mslab.enabled</name> + <value>true</value> + <description> + Enables the MemStore-Local Allocation Buffer, + a feature which works to prevent heap fragmentation under + heavy write loads. This can reduce the frequency of stop-the-world + GC pauses on large heaps. + </description> + </property> + <property> + <name>hbase.hregion.max.filesize</name> + <value>10737418240</value> + <description> + Maximum HStoreFile size. If any one of a column families' HStoreFiles has + grown to exceed this value, the hosting HRegion is split in two. + Default: 1G. + </description> + </property> + <property> + <name>hbase.client.scanner.caching</name> + <value>100</value> + <description>Number of rows that will be fetched when calling next + on a scanner if it is not served from (local, client) memory. Higher + caching values will enable faster scanners but will eat up more memory + and some calls of next may take longer and longer times when the cache is empty. + Do not set this value such that the time between invocations is greater + than the scanner timeout; i.e. hbase.regionserver.lease.period + </description> + </property> + <property> + <name>zookeeper.session.timeout</name> + <value>30000</value> + <description>ZooKeeper session timeout. + HBase passes this to the zk quorum as suggested maximum time for a + session (This setting becomes zookeeper's 'maxSessionTimeout'). See + http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions + "The client sends a requested timeout, the server responds with the + timeout that it can give the client. " In milliseconds. + </description> + </property> + <property> + <name>hbase.client.keyvalue.maxsize</name> + <value>10485760</value> + <description>Specifies the combined maximum allowed size of a KeyValue + instance. This is to set an upper boundary for a single entry saved in a + storage file. Since they cannot be split it helps avoiding that a region + cannot be split any further because the data is too large. It seems wise + to set this to a fraction of the maximum region size. Setting it to zero + or less disables the check. + </description> + </property> + <property> + <name>hbase.hstore.compactionThreshold</name> + <value>3</value> + <description> + If more than this number of HStoreFiles in any one HStore + (one HStoreFile is written per flush of memstore) then a compaction + is run to rewrite all HStoreFiles files as one. Larger numbers + put off compaction but when it runs, it takes longer to complete. + </description> + </property> + <property> + <name>hbase.hstore.flush.retries.number</name> + <value>120</value> + <description> + The number of times the region flush operation will be retried. + </description> + </property> + + <property> + <name>hbase.hstore.blockingStoreFiles</name> + <value>10</value> + <description> + If more than this number of StoreFiles in any one Store + (one StoreFile is written per flush of MemStore) then updates are + blocked for this HRegion until a compaction is completed, or + until hbase.hstore.blockingWaitTime has been exceeded. + </description> + </property> + <property> + <name>hfile.block.cache.size</name> + <value>0.40</value> + <description> + Percentage of maximum heap (-Xmx setting) to allocate to block cache + used by HFile/StoreFile. Default of 0.25 means allocate 25%. + Set to 0 to disable but it's not recommended. + </description> + </property> + + <!-- The following properties configure authentication information for + HBase processes when using Kerberos security. There are no default + values, included here for documentation purposes --> + <property> + <name>hbase.master.keytab.file</name> + <value>/etc/security/keytabs/hbase.service.keytab</value> + <description>Full path to the kerberos keytab file to use for logging in + the configured HMaster server principal. + </description> + </property> + <property> + <name>hbase.master.kerberos.principal</name> + <value>hbase/_h...@example.com</value> + <description>Ex. "hbase/_h...@example.com". The kerberos principal name + that should be used to run the HMaster process. The principal name should + be in the form: user/hostname@DOMAIN. If "_HOST" is used as the hostname + portion, it will be replaced with the actual hostname of the running + instance. + </description> + </property> + <property> + <name>hbase.regionserver.keytab.file</name> + <value>/etc/security/keytabs/hbase.service.keytab</value> + <description>Full path to the kerberos keytab file to use for logging in + the configured HRegionServer server principal. + </description> + </property> + <property> + <name>hbase.regionserver.kerberos.principal</name> + <value>hbase/_h...@example.com</value> + <description>Ex. "hbase/_h...@example.com". The kerberos principal name + that should be used to run the HRegionServer process. The principal name + should be in the form: user/hostname@DOMAIN. If "_HOST" is used as the + hostname portion, it will be replaced with the actual hostname of the + running instance. An entry for this principal must exist in the file + specified in hbase.regionserver.keytab.file + </description> + </property> + + <!-- Additional configuration specific to HBase security --> + <property> + <name>hbase.superuser</name> + <value>hbase</value> + <description>List of users or groups (comma-separated), who are allowed + full privileges, regardless of stored ACLs, across the cluster. + Only used when HBase security is enabled. + </description> + </property> + + <property> + <name>hbase.security.authentication</name> + <value>simple</value> + <description> Controls whether or not secure authentication is enabled for HBase. Possible values are 'simple' + (no authentication), and 'kerberos'. + </description> + </property> + + <property> + <name>hbase.security.authorization</name> + <value>false</value> + <description>Enables HBase authorization. Set the value of this property to false to disable HBase authorization. + </description> + </property> + + <property> + <name>hbase.coprocessor.region.classes</name> + <value></value> + <description>A comma-separated list of Coprocessors that are loaded by + default on all tables. For any override coprocessor method, these classes + will be called in order. After implementing your own Coprocessor, just put + it in HBase's classpath and add the fully qualified class name here. + A coprocessor can also be loaded on demand by setting HTableDescriptor. + </description> + </property> + + <property> + <name>hbase.coprocessor.master.classes</name> + <value></value> + <description>A comma-separated list of + org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are + loaded by default on the active HMaster process. For any implemented + coprocessor methods, the listed classes will be called in order. After + implementing your own MasterObserver, just put it in HBase's classpath + and add the fully qualified class name here. + </description> + </property> + + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>2181</value> + <description>Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + </description> + </property> + + <!-- + The following three properties are used together to create the list of + host:peer_port:leader_port quorum servers for ZooKeeper. + --> + <property> + <name>hbase.zookeeper.quorum</name> + <value>localhost</value> + <description>Comma separated list of servers in the ZooKeeper Quorum. + For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com". + By default this is set to localhost for local and pseudo-distributed modes + of operation. For a fully-distributed setup, this should be set to a full + list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh + this is the list of servers which we will start/stop ZooKeeper on. + </description> + </property> + <!-- End of properties used to generate ZooKeeper host:port quorum list. --> + + <property> + <name>hbase.zookeeper.useMulti</name> + <value>true</value> + <description>Instructs HBase to make use of ZooKeeper's multi-update functionality. + This allows certain ZooKeeper operations to complete more quickly and prevents some issues + with rare Replication failure scenarios (see the release note of HBASE-2611 for an example).· + IMPORTANT: only set this to true if all ZooKeeper servers in the cluster are on version 3.4+ + and will not be downgraded. ZooKeeper versions before 3.4 do not support multi-update and will + not fail gracefully if multi-update is invoked (see ZOOKEEPER-1495). + </description> + </property> + <property> + <name>zookeeper.znode.parent</name> + <value>/hbase-unsecure</value> + <description>Root ZNode for HBase in ZooKeeper. All of HBase's ZooKeeper + files that are configured with a relative path will go under this node. + By default, all of HBase's ZooKeeper file path are configured with a + relative path, so they will all go under this directory unless changed. + </description> + </property> + + <property> + <name>hbase.defaults.for.version.skip</name> + <value>true</value> + <description>Disables version verification.</description> + </property> + + <property> + <name>dfs.domain.socket.path</name> + <value>/var/lib/hadoop-hdfs/dn_socket</value> + <description>Path to domain socket.</description> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/jmx_metrics.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/jmx_metrics.json b/app-packages/hbase-win/jmx_metrics.json new file mode 100644 index 0000000..ac0640e --- /dev/null +++ b/app-packages/hbase-win/jmx_metrics.json @@ -0,0 +1,56 @@ +{ + "Component": { + "HBASE_MASTER": { + "MetricAverageLoad": { + "metric": "Hadoop:service=HBase,name=Master,sub=Server.averageLoad", + "pointInTime": true, + "temporal": false + }, + "DeadRegionServers": { + "metric": "Hadoop:service=HBase,name=Master,sub=Server.numDeadRegionServers", + "pointInTime": true, + "temporal": false + }, + "ClusterId": { + "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.clusterId", + "pointInTime": true, + "temporal": false + }, + "IsActiveMaster": { + "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.isActiveMaster", + "pointInTime": true, + "temporal": false + }, + "MasterActiveTime": { + "metric": "Hadoop:service=HBase,name=Master,sub=Server.masterActiveTime", + "pointInTime": true, + "temporal": false + }, + "MasterStartTime": { + "metric": "Hadoop:service=HBase,name=Master,sub=Server.masterStartTime", + "pointInTime": true, + "temporal": false + }, + "RegionServers": { + "metric": "Hadoop:service=HBase,name=Master,sub=Server.numRegionServers", + "pointInTime": true, + "temporal": false + }, + "ServerName": { + "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.serverName", + "pointInTime": true, + "temporal": false + }, + "ZookeeperQuorum": { + "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.zookeeperQuorum", + "pointInTime": true, + "temporal": false + }, + "ClusterRequests": { + "metric": "Hadoop:service=HBase,name=Master,sub=Server.clusterRequests", + "pointInTime": true, + "temporal": false + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/metainfo.xml b/app-packages/hbase-win/metainfo.xml new file mode 100644 index 0000000..4ab56ef --- /dev/null +++ b/app-packages/hbase-win/metainfo.xml @@ -0,0 +1,119 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <application> + <name>HBASE</name> + <comment> + Apache HBase is the Hadoop database, a distributed, scalable, big data store. + Requirements: + 1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner. + 2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance. + </comment> + <version>${hbase.version}</version> + <type>YARN-APP</type> + <minHadoopVersion>2.1.0</minHadoopVersion> + <exportedConfigs>hbase-site</exportedConfigs> + <exportGroups> + <exportGroup> + <name>QuickLinks</name> + <exports> + <export> + <name>org.apache.slider.jmx</name> + <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value> + </export> + <export> + <name>org.apache.slider.monitor</name> + <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value> + </export> + </exports> + </exportGroup> + </exportGroups> + <commandOrders> + <commandOrder> + <command>HBASE_REGIONSERVER-START</command> + <requires>HBASE_MASTER-STARTED</requires> + </commandOrder> + <commandOrder> + <command>HBASE_MASTER-START</command> + <requires>HBASE_REST-INSTALLED</requires> + </commandOrder> + </commandOrders> + <components> + <component> + <name>HBASE_MASTER</name> + <category>MASTER</category> + <minInstanceCount>1</minInstanceCount> + <maxInstanceCount>2</maxInstanceCount> + <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor</appExports> + <componentExports> + <componentExport> + <name>app.jmx</name> + <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value> + </componentExport> + <componentExport> + <name>app.monitor</name> + <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value> + </componentExport> + </componentExports> + <commandScript> + <script>scripts/hbase_master.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>HBASE_REGIONSERVER</name> + <category>SLAVE</category> + <minInstanceCount>1</minInstanceCount> + <commandScript> + <script>scripts/hbase_regionserver.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> + + </components> + + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>zip</type> + <name>files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <configFiles> + <configFile> + <type>xml</type> + <fileName>hbase-site.xml</fileName> + <dictionaryName>hbase-site</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>hbase-log4j.properties</fileName> + <dictionaryName>hbase-log4j</dictionaryName> + </configFile> + </configFiles> + + </application> +</metainfo> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME b/app-packages/hbase-win/package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME new file mode 100644 index 0000000..96ed9f1 --- /dev/null +++ b/app-packages/hbase-win/package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Replace with the actual HBase package for Windows. + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/__init__.py b/app-packages/hbase-win/package/scripts/__init__.py new file mode 100644 index 0000000..5561e10 --- /dev/null +++ b/app-packages/hbase-win/package/scripts/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/scripts/functions.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/functions.py b/app-packages/hbase-win/package/scripts/functions.py new file mode 100644 index 0000000..e6e7fb9 --- /dev/null +++ b/app-packages/hbase-win/package/scripts/functions.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +import re +import math +import datetime + +from resource_management.core.shell import checked_call + +def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max): + """ + @param heapsize_str: str (e.g '1000m') + @param xmn_percent: float (e.g 0.2) + @param xmn_max: integer (e.g 512) + """ + heapsize = int(re.search('\d+',heapsize_str).group(0)) + heapsize_unit = re.search('\D+',heapsize_str).group(0) + xmn_val = int(math.floor(heapsize*xmn_percent)) + xmn_val -= xmn_val % 8 + + result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val + return str(result_xmn_val) + heapsize_unit http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase.py b/app-packages/hbase-win/package/scripts/hbase.py new file mode 100644 index 0000000..70601bc --- /dev/null +++ b/app-packages/hbase-win/package/scripts/hbase.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os + +from resource_management import * +import sys +import shutil + +def hbase(name=None # 'master' or 'regionserver' or 'client' + ): + import params + + if name in ["master","regionserver"]: + params.HdfsDirectory(params.hbase_hdfs_root_dir, + action="create_delayed" + ) + params.HdfsDirectory(params.hbase_staging_dir, + action="create_delayed", + mode=0711 + ) + params.HdfsDirectory(None, action="create") + + Directory( params.conf_dir, + owner = params.hbase_user, + group = params.user_group, + recursive = True, + content = params.input_conf_files_dir + ) + + Directory (params.tmp_dir, + owner = params.hbase_user, + recursive = True + ) + + Directory (os.path.join(params.local_dir, "jars"), + owner = params.hbase_user, + group = params.user_group, + mode=0775, + recursive = True + ) + + XmlConfig( "hbase-site.xml", + conf_dir = params.conf_dir, + configurations = params.config['configurations']['hbase-site'], + owner = params.hbase_user, + group = params.user_group + ) + + + if (params.log4j_props != None): + File(format("{params.conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hbase_user, + content=params.log4j_props + ) + elif (os.path.exists(format("{conf_dir}/log4j.properties"))): + File(format("{params.conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hbase_user + ) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_master.py b/app-packages/hbase-win/package/scripts/hbase_master.py new file mode 100644 index 0000000..f4e202f --- /dev/null +++ b/app-packages/hbase-win/package/scripts/hbase_master.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service + + +class HbaseMaster(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='master') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + ##hbase_service( 'master', + ## action = 'start' + ##) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'master', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid") + ##check_process_status(pid_file) + + +if __name__ == "__main__": + HbaseMaster().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_regionserver.py b/app-packages/hbase-win/package/scripts/hbase_regionserver.py new file mode 100644 index 0000000..7114dc8 --- /dev/null +++ b/app-packages/hbase-win/package/scripts/hbase_regionserver.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service + + +class HbaseRegionServer(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='regionserver') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + ##hbase_service( 'regionserver', + ## action = 'start' + ##) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'regionserver', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid") + ##check_process_status(pid_file) + + +if __name__ == "__main__": + HbaseRegionServer().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/scripts/hbase_service.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_service.py b/app-packages/hbase-win/package/scripts/hbase_service.py new file mode 100644 index 0000000..96add84 --- /dev/null +++ b/app-packages/hbase-win/package/scripts/hbase_service.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +def hbase_service( + name, + action = 'start'): # 'start' or 'stop' or 'status' + + import params + + role = name + cmd = format("{daemon_script} --config {conf_dir}") + pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid") + + daemon_cmd = None + no_op_test = None + + if action == 'start': + daemon_cmd = format("{cmd} start {role}") + if name == 'rest': + daemon_cmd = format("{daemon_cmd} -p {rest_port}") + elif name == 'thrift': + daemon_cmd = format("{daemon_cmd} -p {thrift_port}") + elif name == 'thrift2': + daemon_cmd = format("{daemon_cmd} -p {thrift2_port}") + no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + elif action == 'stop': + daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}") + + if daemon_cmd is not None: + Execute ( daemon_cmd, + not_if = no_op_test + ) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/params.py b/app-packages/hbase-win/package/scripts/params.py new file mode 100644 index 0000000..33e2bbf --- /dev/null +++ b/app-packages/hbase-win/package/scripts/params.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from functions import calc_xmn_from_xms +from resource_management import * +import status_params + +# server configurations +config = Script.get_config() + +hbase_root = config['configurations']['global']['app_root'] +conf_dir = format("{hbase_root}/conf") +daemon_script = format("{hbase_root}/bin/hbase-daemon.sh") + +hbase_user = status_params.hbase_user +_authentication = config['configurations']['core-site']['hadoop.security.authentication'] +security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +user_group = config['configurations']['global']['user_group'] + +# this is "hadoop-metrics.properties" for 1.x stacks +metric_prop_file_name = "hadoop-metrics2-hbase.properties" + +# not supporting 32 bit jdk. +java64_home = config['hostLevelParams']['java_home'] + +log_dir = config['configurations']['global']['app_log_dir'] +master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize'] + +regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize'] +regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max'] +regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio'] +regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) + +pid_dir = status_params.pid_dir +tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] +local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'], config['configurations']['hbase-site']) +input_conf_files_dir = config['configurations']['global']['app_input_conf_dir'] + +client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf_dir}/hbase_client_jaas.conf")) +master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf")) +regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', format("{conf_dir}/hbase_regionserver_jaas.conf")) + +ganglia_server_host = default('/configurations/global/ganglia_server_host', '') +ganglia_server_port = default('/configurations/global/ganglia_server_port', '8663') + +rest_port = config['configurations']['global']['hbase_rest_port'] +thrift_port = config['configurations']['global']['hbase_thrift_port'] +thrift2_port = config['configurations']['global']['hbase_thrift2_port'] + +if security_enabled: + _hostname_lowercase = config['hostname'].lower() + master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase) + regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase) + + +master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] +regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] +kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +if security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};") +else: + kinit_cmd = "" + +#log4j.properties +if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])): + log4j_props = config['configurations']['hbase-log4j']['content'] +else: + log4j_props = None + +hbase_env_sh_template = config['configurations']['hbase-env']['content'] + +hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir'] +hbase_staging_dir = config['configurations']['hbase-site']['hbase.stagingdir'] +#for create_hdfs_directory +hostname = config["hostname"] +hadoop_conf_dir = "/etc/hadoop/conf" +hdfs_user_keytab = config['configurations']['global']['hdfs_user_keytab'] +hdfs_user = config['configurations']['global']['hdfs_user'] +kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +import functools +#create partial functions with common arguments for every HdfsDirectory call +#to create hdfs directory we need to call params.HdfsDirectory in code +HdfsDirectory = functools.partial( + HdfsDirectory, + conf_dir=hadoop_conf_dir, + hdfs_user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local +) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/status_params.py b/app-packages/hbase-win/package/scripts/status_params.py new file mode 100644 index 0000000..c18cbb9 --- /dev/null +++ b/app-packages/hbase-win/package/scripts/status_params.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +config = Script.get_config() + +pid_dir = config['configurations']['global']['app_pid_dir'] +hbase_user = config['configurations']['global']['app_user'] http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/templates/hbase_client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/templates/hbase_client_jaas.conf.j2 b/app-packages/hbase-win/package/templates/hbase_client_jaas.conf.j2 new file mode 100644 index 0000000..bb4279c --- /dev/null +++ b/app-packages/hbase-win/package/templates/hbase_client_jaas.conf.j2 @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=false +useTicketCache=true; +}; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/templates/hbase_master_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/templates/hbase_master_jaas.conf.j2 b/app-packages/hbase-win/package/templates/hbase_master_jaas.conf.j2 new file mode 100644 index 0000000..91ce3ef --- /dev/null +++ b/app-packages/hbase-win/package/templates/hbase_master_jaas.conf.j2 @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{master_keytab_path}}" +principal="{{master_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/package/templates/hbase_regionserver_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/templates/hbase_regionserver_jaas.conf.j2 b/app-packages/hbase-win/package/templates/hbase_regionserver_jaas.conf.j2 new file mode 100644 index 0000000..2a9b9f3 --- /dev/null +++ b/app-packages/hbase-win/package/templates/hbase_regionserver_jaas.conf.j2 @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{regionserver_keytab_path}}" +principal="{{regionserver_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/resources.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/resources.json b/app-packages/hbase-win/resources.json new file mode 100644 index 0000000..e0ff26f --- /dev/null +++ b/app-packages/hbase-win/resources.json @@ -0,0 +1,21 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { + }, + "components": { + "HBASE_MASTER": { + "yarn.role.priority": "1", + "yarn.component.instances": "1", + "yarn.memory": "256" + }, + "slider-appmaster": { + }, + "HBASE_REGIONSERVER": { + "yarn.role.priority": "2", + "yarn.component.instances": "1", + "yarn.memory": "256" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/slider-agent/src/main/python/resource_management/core/providers/windows/system.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py index f0d4825..3165b5d 100644 --- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py +++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py @@ -28,13 +28,59 @@ import time import os import subprocess import shutil - - -def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None, pid_file_name=None): - # TODO implement logoutput +from resource_management.libraries.script import Script +import win32con +from win32security import * +from win32api import * +from winerror import ERROR_INVALID_HANDLE +from win32profile import CreateEnvironmentBlock +from win32process import GetExitCodeProcess, STARTF_USESTDHANDLES, STARTUPINFO, CreateProcessAsUser +from win32event import WaitForSingleObject, INFINITE +import msvcrt +import tempfile + + +def _merge_env(env1, env2, merge_keys=['PYTHONPATH']): + """ + Merge env2 into env1. Also current python instance variables from merge_keys list taken into account and they will be + merged with equivalent keys from env1 and env2 using system path separator. + :param env1: first environment, usually returned by CreateEnvironmentBlock + :param env2: custom environment + :param merge_keys: env variables to merge as PATH + :return: merged environment + """ + env1 = dict(env1) # copy to new dict in case env1 is os.environ + if env2: + for key, value in env2.iteritems(): + if not key in merge_keys: + env1[key] = value + # strnsform keys and values to str(windows can not accept unicode) + result_env = {} + for key, value in env1.iteritems(): + if not key in merge_keys: + result_env[str(key)] = str(value) + #merge keys from merge_keys + def put_values(key, env, result): + if env and key in env: + result.extend(env[key].split(os.pathsep)) + + for key in merge_keys: + all_values = [] + for env in [env1, env2, os.environ]: + put_values(key, env, all_values) + result_env[str(key)] = str(os.pathsep.join(set(all_values))) + return result_env + +# Execute command. As windows stack heavily relies on proper environment it is better to reload fresh environment +# on every execution. env variable will me merged with fresh environment for user. +def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None, user=None, pid_file_name=None): + # TODO implement user Logger.info("Executing %s" % (command)) + cur_token = OpenProcessToken(GetCurrentProcess(), TOKEN_ALL_ACCESS) + current_env = CreateEnvironmentBlock(cur_token, False) + current_env = _merge_env(current_env, env) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - cwd=cwd, env=env, shell=False) + cwd=cwd, env=current_env, shell=False) if not wait_for_finish: if pid_file_name: pidfile = open(pid_file_name, 'w') @@ -47,20 +93,22 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish= t = threading.Timer( timeout, on_timeout, [proc, q] ) t.start() - out = proc.communicate()[0].strip() - code = proc.returncode + out, err = proc.communicate() + code = proc.returncode if logoutput and out: Logger.info(out) - return code, out + if logoutput and err: + Logger.info(err) + return code, out, err # see msdn Icacls doc for rights def _set_file_acl(file, user, rights): acls_modify_cmd = "icacls {0} /grant {1}:{2}".format(file, user, rights) acls_remove_cmd = "icacls {0} /remove {1}".format(file, user) - code, out = _call_command(acls_remove_cmd) + code, out, err = _call_command(acls_remove_cmd) if code != 0: raise Fail("Can not remove rights for path {0} and user {1}".format(file, user)) - code, out = _call_command(acls_modify_cmd) + code, out, err = _call_command(acls_modify_cmd) if code != 0: raise Fail("Can not set rights {0} for path {1} and user {2}".format(file, user)) else: @@ -137,10 +185,13 @@ class ExecuteProvider(Provider): for i in range(0, self.resource.tries): try: - _call_command(self.resource.command, logoutput=self.resource.logoutput, - cwd=self.resource.cwd, env=self.resource.environment, - wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout, - pid_file_name=self.resource.pid_file) + code, _, _ = _call_command(self.resource.command, logoutput=self.resource.logoutput, + cwd=self.resource.cwd, env=self.resource.environment, + wait_for_finish=self.resource.wait_for_finish, + timeout=self.resource.timeout, user=self.resource.user, + pid_file_name=self.resource.pid_file) + if code != 0 and not self.resource.ignore_failures: + raise Fail("Failed to execute " + self.resource.command) break except Fail as ex: if i == self.resource.tries - 1: # last try http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index ff71151..669a19a 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -1380,6 +1380,7 @@ public class AgentProviderService extends AbstractProviderService implements List<ConfigFile> configFiles = getMetainfo().getApplication().getConfigFiles(); for(ConfigFile configFile : configFiles) { + log.info("Expecting config type {}.", configFile.getDictionaryName()); configList.add(configFile.getDictionaryName()); } @@ -1419,6 +1420,7 @@ public class AgentProviderService extends AbstractProviderService implements //apply defaults only if the key is not present and value is not empty if(getDefaultConfigs().containsKey(configName)) { + log.info("Adding default configs for type {}.", configName); for(PropertyInfo defaultConfigProp : getDefaultConfigs().get(configName).getPropertyInfos()) { if(!config.containsKey(defaultConfigProp.getName())){ if(!defaultConfigProp.getName().isEmpty() &&
