http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/accumulo-watcher.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/accumulo-watcher.sh b/assemble/scripts/accumulo-watcher.sh new file mode 100755 index 0000000..9fb13a2 --- /dev/null +++ b/assemble/scripts/accumulo-watcher.sh @@ -0,0 +1,133 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOGHOST=$1 +shift +process=$1 + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +# Stop: Resolve Script Directory + +. "${bin}"/config.sh + +CLEAN_EXIT="Clean Exit" +UNEXPECTED_EXCEPTION="Unexpected exception" +OOM_EXCEPTION="Out of memory exception" +ZKLOCK_LOST="ZKLock lost" +UNKNOWN_ERROR="Unknown error" + +ERRFILE=${ACCUMULO_LOG_DIR}/${process}_${LOGHOST}.err +OUTFILE=${ACCUMULO_LOG_DIR}/${process}_${LOGHOST}.out +DEBUGLOG=${ACCUMULO_LOG_DIR}/${process}_$(hostname).debug.log +export COMMAND="${bin}/accumulo \"\$@\"" + +logger -s "starting process $process at $(date)" +stopRunning="" +while [ -z "$stopRunning" ]; +do + eval $COMMAND 2> $ERRFILE + exit=$? + unset cause + if [ "$exit" -eq 0 ]; then + potentialStopRunning=$CLEAN_EXIT + elif [ "$exit" -eq 1 ]; then + potentialStopRunning=$UNEXPECTED_EXCEPTION + elif [ "$exit" -eq 130 ]; then + stopRunning="Control C detected, exiting" + elif [ "$exit" -eq 143 ]; then + stopRunning="Process terminated, exiting" + elif [ "$exit" -eq 137 ]; then + potentialStopRunning="Process killed, exiting" + fi + if [ -z "$stopRunning" ]; then + stopRunning=$potentialStopRunning; + + if [ $exit -eq 1 ]; then + source="exit code" + cause=$UNEXPECTED_EXCEPTION + elif tail -n50 $OUTFILE | grep "java.lang.OutOfMemoryError:" > /dev/null; then + source="logs" + cause=$OOM_EXCEPTION + elif [ "$process" = "tserver" ]; then + if tail -n50 $DEBUGLOG | grep "ERROR: Lost tablet server lock (reason =" > /dev/null ; then + source="logs" + cause=$ZKLOCK_LOST + fi + elif [ "$process" = "master" ]; then + if tail -n50 $DEBUGLOG | grep "ERROR: Master lock in zookeeper lost (reason =" > /dev/null ; then + source="logs" + cause=$ZKLOCK_LOST + fi + elif [ "$process" = "gc" ]; then + if tail -n50 $DEBUGLOG | grep "FATAL: GC lock in zookeeper lost (reason =" > /dev/null ; then + source="logs" + cause=$ZKLOCK_LOST + fi + elif [ "$process" = "monitor" ]; then + if tail -n50 $DEBUGLOG | grep "ERROR: Monitor lock in zookeeper lost (reason =" > /dev/null ; then + source="logs" + cause=$ZKLOCK_LOST + fi + elif [ $exit -ne 0 ]; then + source="exit code" + cause=$UNKNOWN_ERROR + fi + case $cause in + #Unknown exit code + "$UNKNOWN_ERROR") + #window doesn't matter when retries = 0 + RETRIES=0 + ;; + + "$UNEXPECTED_EXCEPTION") + WINDOW=$UNEXPECTED_TIMESPAN + RETRIES=$UNEXPECTED_RETRIES + ;; + + "$OOM_EXCEPTION") + WINDOW=$OOM_TIMESPAN + RETRIES=$OOM_RETRIES + ;; + + "$ZLOCK_LOST") + WINDOW=$ZKLOCK_TIMESPAN + RETRIES=$ZKLOCK_RETRIES + ;; + esac + + if [ -n "$cause" ]; then + stopRunning="" + declare -i attempts + attempts="`jobs | grep "reason$cause" | wc -l`+1" + if [ "$RETRIES" -le $attempts ]; then + stopRunning="$process encountered $cause in $source with exit code $exit- quitting ($attempts/$RETRIES in $WINDOW seconds)" + # kill all sleeps now + for list in `jobs | cut -b 2-2`; do kill %$list; done + else + logger -s "$process encountered $cause in $source with exit code $exit- retrying ($attempts/$RETRIES in $WINDOW seconds)" + eval "(sleep $WINDOW ; echo "reason$cause" >> /dev/null) &" + fi + fi + fi +done +logger -s $stopRunning
http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/bootstrap-config.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/bootstrap-config.sh b/assemble/scripts/bootstrap-config.sh new file mode 100755 index 0000000..44ae2ac --- /dev/null +++ b/assemble/scripts/bootstrap-config.sh @@ -0,0 +1,407 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function usage { + cat <<EOF +Usage: bootstrap_config.sh [-options] +where options include (long options not available on all platforms): + -d, --dir Alternate directory to setup config files + -s, --size Supported sizes: '1GB' '2GB' '3GB' '512MB' + -n, --native Configure to use native libraries + -j, --jvm Configure to use the jvm + -o, --overwrite Overwrite the default config directory + -v, --version Specify the Apache Hadoop version supported versions: '1' '2' + -k, --kerberos Configure for use with Kerberos + -h, --help Print this help message +EOF +} + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +# Stop: Resolve Script Directory + +# +# Resolve accumulo home for bootstrapping +# +ACCUMULO_HOME=$( cd -P ${bin}/.. && pwd ) +TEMPLATE_CONF_DIR="${ACCUMULO_HOME}/conf/templates" +CONF_DIR="${ACCUMULO_HOME}/conf" +ACCUMULO_SITE=accumulo-site.xml +ACCUMULO_ENV=accumulo-env.sh + +SIZE= +TYPE= +HADOOP_VERSION= +OVERWRITE="0" +BASE_DIR= +KERBEROS= + +#Execute getopt +if [[ $(uname -s) == "Linux" ]]; then + args=$(getopt -o "b:d:s:njokv:h" -l "basedir:,dir:,size:,native,jvm,overwrite,kerberos,version:,help" -q -- "$@") +else # Darwin, BSD + args=$(getopt b:d:s:njokv:h $*) +fi + +#Bad arguments +if [[ $? != 0 ]]; then + usage 1>&2 + exit 1 +fi +eval set -- $args + +for i +do + case "$i" in + -b|--basedir) #Hidden option used to set general.maven.project.basedir for developers + BASE_DIR=$2; shift + shift;; + -d|--dir) + CONF_DIR=$2; shift + shift;; + -s|--size) + SIZE=$2; shift + shift;; + -n|--native) + TYPE=native + shift;; + -j|--jvm) + TYPE=jvm + shift;; + -o|--overwrite) + OVERWRITE=1 + shift;; + -v|--version) + HADOOP_VERSION=$2; shift + shift;; + -k|--kerberos) + KERBEROS="true" + shift;; + -h|--help) + usage + exit 0 + shift;; + --) + shift + break;; + esac +done + +while [[ "${OVERWRITE}" = "0" ]]; do + if [[ -e "${CONF_DIR}/${ACCUMULO_ENV}" || -e "${CONF_DIR}/${ACCUMULO_SITE}" ]]; then + echo "Warning your current config files in ${CONF_DIR} will be overwritten!" + echo + echo "How would you like to proceed?:" + select CHOICE in 'Continue with overwrite' 'Specify new conf dir'; do + if [[ "${CHOICE}" = 'Specify new conf dir' ]]; then + echo -n "Please specifiy new conf directory: " + read CONF_DIR + elif [[ "${CHOICE}" = 'Continue with overwrite' ]]; then + OVERWRITE=1 + fi + break + done + else + OVERWRITE=1 + fi +done +echo "Copying configuration files to: ${CONF_DIR}" + +#Native 1GB +native_1GB_tServer="-Xmx128m -Xms128m" +_1GB_master="-Xmx128m -Xms128m" +_1GB_monitor="-Xmx64m -Xms64m" +_1GB_gc="-Xmx64m -Xms64m" +_1GB_other="-Xmx128m -Xms64m" +_1GB_shell="${_1GB_other}" + +_1GB_memoryMapMax="256M" +native_1GB_nativeEnabled="true" +_1GB_cacheDataSize="15M" +_1GB_cacheIndexSize="40M" +_1GB_sortBufferSize="50M" +_1GB_waLogMaxSize="256M" + +#Native 2GB +native_2GB_tServer="-Xmx256m -Xms256m" +_2GB_master="-Xmx256m -Xms256m" +_2GB_monitor="-Xmx128m -Xms64m" +_2GB_gc="-Xmx128m -Xms128m" +_2GB_other="-Xmx256m -Xms64m" +_2GB_shell="${_2GB_other}" + +_2GB_memoryMapMax="512M" +native_2GB_nativeEnabled="true" +_2GB_cacheDataSize="30M" +_2GB_cacheIndexSize="80M" +_2GB_sortBufferSize="50M" +_2GB_waLogMaxSize="512M" + +#Native 3GB +native_3GB_tServer="-Xmx1g -Xms1g -XX:NewSize=500m -XX:MaxNewSize=500m" +_3GB_master="-Xmx1g -Xms1g" +_3GB_monitor="-Xmx1g -Xms256m" +_3GB_gc="-Xmx256m -Xms256m" +_3GB_other="-Xmx1g -Xms256m" +_3GB_shell="${_3GB_other}" + +_3GB_memoryMapMax="1G" +native_3GB_nativeEnabled="true" +_3GB_cacheDataSize="128M" +_3GB_cacheIndexSize="128M" +_3GB_sortBufferSize="200M" +_3GB_waLogMaxSize="1G" + +#Native 512MB +native_512MB_tServer="-Xmx48m -Xms48m" +_512MB_master="-Xmx128m -Xms128m" +_512MB_monitor="-Xmx64m -Xms64m" +_512MB_gc="-Xmx64m -Xms64m" +_512MB_other="-Xmx128m -Xms64m" +_512MB_shell="${_512MB_other}" + +_512MB_memoryMapMax="80M" +native_512MB_nativeEnabled="true" +_512MB_cacheDataSize="7M" +_512MB_cacheIndexSize="20M" +_512MB_sortBufferSize="50M" +_512MB_waLogMaxSize="100M" + +#JVM 1GB +jvm_1GB_tServer="-Xmx384m -Xms384m" + +jvm_1GB_nativeEnabled="false" + +#JVM 2GB +jvm_2GB_tServer="-Xmx768m -Xms768m" + +jvm_2GB_nativeEnabled="false" + +#JVM 3GB +jvm_3GB_tServer="-Xmx2g -Xms2g -XX:NewSize=1G -XX:MaxNewSize=1G" + +jvm_3GB_nativeEnabled="false" + +#JVM 512MB +jvm_512MB_tServer="-Xmx128m -Xms128m" + +jvm_512MB_nativeEnabled="false" + + +if [[ -z "${SIZE}" ]]; then + echo "Choose the heap configuration:" + select DIRNAME in 1GB 2GB 3GB 512MB; do + echo "Using '${DIRNAME}' configuration" + SIZE=${DIRNAME} + break + done +elif [[ "${SIZE}" != "1GB" && "${SIZE}" != "2GB" && "${SIZE}" != "3GB" && "${SIZE}" != "512MB" ]]; then + echo "Invalid memory size" + echo "Supported sizes: '1GB' '2GB' '3GB' '512MB'" + exit 1 +fi + +if [[ -z "${TYPE}" ]]; then + echo + echo "Choose the Accumulo memory-map type:" + select TYPENAME in Java Native; do + if [[ "${TYPENAME}" == "Native" ]]; then + TYPE="native" + echo "Don't forget to build the native libraries using the bin/build_native_library.sh script" + elif [[ "${TYPENAME}" == "Java" ]]; then + TYPE="jvm" + fi + echo "Using '${TYPE}' configuration" + echo + break + done +fi + +if [[ -z "${HADOOP_VERSION}" ]]; then + echo + echo "Choose the Apache Hadoop version:" + select HADOOP in 'Hadoop 2' 'HDP 2.0/2.1' 'HDP 2.2' 'IOP 4.1'; do + if [ "${HADOOP}" == "Hadoop 2" ]; then + HADOOP_VERSION="2" + elif [ "${HADOOP}" == "HDP 2.0/2.1" ]; then + HADOOP_VERSION="HDP2" + elif [ "${HADOOP}" == "HDP 2.2" ]; then + HADOOP_VERSION="HDP2.2" + elif [ "${HADOOP}" == "IOP 4.1" ]; then + HADOOP_VERSION="IOP4.1" + fi + echo "Using Hadoop version '${HADOOP_VERSION}' configuration" + echo + break + done +elif [[ "${HADOOP_VERSION}" != "2" && "${HADOOP_VERSION}" != "HDP2" && "${HADOOP_VERSION}" != "HDP2.2" ]]; then + echo "Invalid Hadoop version" + echo "Supported Hadoop versions: '2', 'HDP2', 'HDP2.2'" + exit 1 +fi + +TRACE_USER="root" + +if [[ ! -z "${KERBEROS}" ]]; then + echo + read -p "Enter server's Kerberos principal: " PRINCIPAL + read -p "Enter server's Kerberos keytab: " KEYTAB + TRACE_USER="${PRINCIPAL}" +fi + +for var in SIZE TYPE HADOOP_VERSION; do + if [[ -z ${!var} ]]; then + echo "Invalid $var configuration" + exit 1 + fi +done + +TSERVER="${TYPE}_${SIZE}_tServer" +MASTER="_${SIZE}_master" +MONITOR="_${SIZE}_monitor" +GC="_${SIZE}_gc" +SHELL="_${SIZE}_shell" +OTHER="_${SIZE}_other" + +MEMORY_MAP_MAX="_${SIZE}_memoryMapMax" +NATIVE="${TYPE}_${SIZE}_nativeEnabled" +CACHE_DATA_SIZE="_${SIZE}_cacheDataSize" +CACHE_INDEX_SIZE="_${SIZE}_cacheIndexSize" +SORT_BUFFER_SIZE="_${SIZE}_sortBufferSize" +WAL_MAX_SIZE="_${SIZE}_waLogMaxSize" + +MAVEN_PROJ_BASEDIR="" + +if [[ ! -z "${BASE_DIR}" ]]; then + MAVEN_PROJ_BASEDIR="\n <property>\n <name>general.maven.project.basedir</name>\n <value>${BASE_DIR}</value>\n </property>\n" +fi + +#Configure accumulo-env.sh +mkdir -p "${CONF_DIR}" && cp ${TEMPLATE_CONF_DIR}/* ${CONF_DIR}/ +sed -e "s/\${tServerHigh_tServerLow}/${!TSERVER}/" \ + -e "s/\${masterHigh_masterLow}/${!MASTER}/" \ + -e "s/\${monitorHigh_monitorLow}/${!MONITOR}/" \ + -e "s/\${gcHigh_gcLow}/${!GC}/" \ + -e "s/\${shellHigh_shellLow}/${!SHELL}/" \ + -e "s/\${otherHigh_otherLow}/${!OTHER}/" \ + ${TEMPLATE_CONF_DIR}/$ACCUMULO_ENV > ${CONF_DIR}/$ACCUMULO_ENV + +#Configure accumulo-site.xml +sed -e "s/\${memMapMax}/${!MEMORY_MAP_MAX}/" \ + -e "s/\${nativeEnabled}/${!NATIVE}/" \ + -e "s/\${cacheDataSize}/${!CACHE_DATA_SIZE}/" \ + -e "s/\${cacheIndexSize}/${!CACHE_INDEX_SIZE}/" \ + -e "s/\${sortBufferSize}/${!SORT_BUFFER_SIZE}/" \ + -e "s/\${waLogMaxSize}/${!WAL_MAX_SIZE}/" \ + -e "s=\${traceUser}=${TRACE_USER}=" \ + -e "s=\${mvnProjBaseDir}=${MAVEN_PROJ_BASEDIR}=" ${TEMPLATE_CONF_DIR}/$ACCUMULO_SITE > ${CONF_DIR}/$ACCUMULO_SITE + +# If we're not using kerberos, filter out the krb properties +if [[ -z "${KERBEROS}" ]]; then + sed -e 's/<!-- Kerberos requirements -->/<!-- Kerberos requirements --><!--/' \ + -e 's/<!-- End Kerberos requirements -->/--><!-- End Kerberos requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" +else + # Make the substitutions + sed -e "s!\${keytab}!${KEYTAB}!" \ + -e "s!\${principal}!${PRINCIPAL}!" \ + ${CONF_DIR}/${ACCUMULO_SITE} > temp + mv temp ${CONF_DIR}/${ACCUMULO_SITE} +fi + +# Configure hadoop version +if [[ "${HADOOP_VERSION}" == "2" ]]; then + sed -e 's/<!-- HDP 2.0 requirements -->/<!-- HDP 2.0 requirements --><!--/' \ + -e 's/<!-- End HDP 2.0 requirements -->/--><!-- End HDP 2.0 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" + sed -e 's/<!-- HDP 2.2 requirements -->/<!-- HDP 2.2 requirements --><!--/' \ + -e 's/<!-- End HDP 2.2 requirements -->/--><!-- End HDP 2.2 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" + sed -e 's/<!-- IOP 4.1 requirements -->/<!-- IOP 4.1 requirements --><!--/' \ + -e 's/<!-- End IOP 4.1 requirements -->/--><!-- End IOP 4.1 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" +elif [[ "${HADOOP_VERSION}" == "HDP2" ]]; then + sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements --><!--/' \ + -e 's/<!-- End Hadoop 2 requirements -->/--><!-- End Hadoop 2 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" + sed -e 's/<!-- HDP 2.2 requirements -->/<!-- HDP 2.2 requirements --><!--/' \ + -e 's/<!-- End HDP 2.2 requirements -->/--><!-- End HDP 2.2 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" + sed -e 's/<!-- IOP 4.1 requirements -->/<!-- IOP 4.1 requirements --><!--/' \ + -e 's/<!-- End IOP 4.1 requirements -->/--><!-- End IOP 4.1 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" +elif [[ "${HADOOP_VERSION}" == "HDP2.2" ]]; then + sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements --><!--/' \ + -e 's/<!-- End Hadoop 2 requirements -->/--><!-- End Hadoop 2 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" + sed -e 's/<!-- HDP 2.0 requirements -->/<!-- HDP 2.0 requirements --><!--/' \ + -e 's/<!-- End HDP 2.0 requirements -->/--><!-- End HDP 2.0 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" + sed -e 's/<!-- IOP 4.1 requirements -->/<!-- IOP 4.1 requirements --><!--/' \ + -e 's/<!-- End IOP 4.1 requirements -->/--><!-- End IOP 4.1 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" +elif [[ "${HADOOP_VERSION}" == "IOP4.1" ]]; then + sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements --><!--/' \ + -e 's/<!-- End Hadoop 2 requirements -->/--><!-- End Hadoop 2 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" + sed -e 's/<!-- HDP 2.0 requirements -->/<!-- HDP 2.0 requirements --><!--/' \ + -e 's/<!-- End HDP 2.0 requirements -->/--><!-- End HDP 2.0 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" + sed -e 's/<!-- HDP 2.2 requirements -->/<!-- HDP 2.2 requirements --><!--/' \ + -e 's/<!-- End HDP 2.2 requirements -->/--><!-- End HDP 2.2 requirements -->/' \ + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" +fi + +#Additional setup steps for native configuration. +if [[ ${TYPE} == native ]]; then + if [[ $(uname) == Linux ]]; then + if [[ -z $HADOOP_PREFIX ]]; then + echo "WARNING: HADOOP_PREFIX not set, cannot automatically configure LD_LIBRARY_PATH to include Hadoop native libraries" + else + NATIVE_LIB=$(readlink -ef $(dirname $(for x in $(find $HADOOP_PREFIX -name libhadoop.so); do ld $x 2>/dev/null && echo $x && break; done) 2>>/dev/null) 2>>/dev/null) + if [[ -z $NATIVE_LIB ]]; then + echo -e "WARNING: The Hadoop native libraries could not be found for your sytem in: $HADOOP_PREFIX" + else + sed "/# Should the monitor/ i export LD_LIBRARY_PATH=${NATIVE_LIB}:\${LD_LIBRARY_PATH}" ${CONF_DIR}/$ACCUMULO_ENV > temp + mv temp "${CONF_DIR}/$ACCUMULO_ENV" + echo -e "Added ${NATIVE_LIB} to the LD_LIBRARY_PATH" + fi + fi + fi + echo -e "Please remember to compile the Accumulo native libraries using the bin/build_native_library.sh script and to set the LD_LIBRARY_PATH variable in the ${CONF_DIR}/accumulo-env.sh script if needed." +fi +echo "Setup complete" http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/bootstrap-hdfs.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/bootstrap-hdfs.sh b/assemble/scripts/bootstrap-hdfs.sh new file mode 100755 index 0000000..1d2964a --- /dev/null +++ b/assemble/scripts/bootstrap-hdfs.sh @@ -0,0 +1,90 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +# Stop: Resolve Script Directory + +. "$bin"/config.sh + +# +# Find the system context directory in HDFS +# +SYSTEM_CONTEXT_HDFS_DIR=$(grep -A1 "general.vfs.classpaths" "$ACCUMULO_CONF_DIR/accumulo-site.xml" | tail -1 | perl -pe 's/\s+<value>//; s/<\/value>//; s/,.+$//; s|[^/]+$||; print $ARGV[1]') + +if [ -z "$SYSTEM_CONTEXT_HDFS_DIR" ] +then + echo "Your accumulo-site.xml file is not set up for the HDFS Classloader. Please add the following to your accumulo-site.xml file where ##CLASSPATH## is one of the following formats:" + echo "A single directory: hdfs://host:port/directory/" + echo "A single directory with a regex: hdfs://host:port/directory/.*.jar" + echo "Multiple directories: hdfs://host:port/directory/.*.jar,hdfs://host:port/directory2/" + echo "" + echo "<property>" + echo " <name>general.vfs.classpaths</name>" + echo " <value>##CLASSPATH##</value>" + echo " <description>location of the jars for the default (system) context</description>" + echo "</property>" + exit 1 +fi + +# +# Create the system context directy in HDFS if it does not exist +# +"$HADOOP_PREFIX/bin/hadoop" fs -ls "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null +if [[ $? != 0 ]]; then + "$HADOOP_PREFIX/bin/hadoop" fs -mkdir "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null + if [[ $? != 0 ]]; then + echo "Unable to create classpath directory at $SYSTEM_CONTEXT_HDFS_DIR" + exit 1 + fi +fi + +# +# Replicate to all tservers to avoid network contention on startup +# +TSERVERS=$ACCUMULO_CONF_DIR/tservers +NUM_TSERVERS=$(egrep -v '(^#|^\s*$)' "$TSERVERS" | wc -l) + +#let each datanode service around 50 clients +REP=$(( NUM_TSERVERS / 50 )) +(( REP < 3 )) && REP=3 + +# +# Copy all jars in lib to the system context directory +# +"$HADOOP_PREFIX/bin/hadoop" fs -moveFromLocal "$ACCUMULO_HOME"/lib/*.jar "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null +"$HADOOP_PREFIX/bin/hadoop" fs -setrep -R $REP "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null + +# +# We need some of the jars in lib, copy them back out and remove them from the system context dir +# +"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2.jar" "$ACCUMULO_HOME/lib/." > /dev/null +"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2.jar" > /dev/null +"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" "$ACCUMULO_HOME/lib/." > /dev/null +"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" > /dev/null +"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/slf4j*.jar" "$ACCUMULO_HOME/lib/." > /dev/null +"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/slf4j*.jar" > /dev/null +for f in $(grep -v '^#' "$ACCUMULO_CONF_DIR/tservers") +do + rsync -ra --delete "$ACCUMULO_HOME" $(dirname "$ACCUMULO_HOME") +done http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/build-native-library.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/build-native-library.sh b/assemble/scripts/build-native-library.sh new file mode 100755 index 0000000..c2531ee --- /dev/null +++ b/assemble/scripts/build-native-library.sh @@ -0,0 +1,74 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink + bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) +script=$( basename "$SOURCE" ) +# Stop: Resolve Script Directory + + +lib=${bin}/../lib +native_tarball=${lib}/accumulo-native.tar.gz +final_native_target="${lib}/native" + +if [[ ! -f $native_tarball ]]; then + echo "Could not find native code artifact: ${native_tarball}"; + exit 1 +fi + +# Make the destination for the native library +mkdir -p "${final_native_target}" || exit 1 + +# Make a directory for us to unpack the native source into +TMP_DIR=$(mktemp -d /tmp/accumulo-native.XXXX) || exit 1 + +# Unpack the tarball to our temp directory +tar xf "${native_tarball}" -C "${TMP_DIR}" + +if [[ $? != 0 ]]; then + echo "Failed to unpack native tarball to ${TMP_DIR}" + exit 1 +fi + +# Move to the first (only) directory in our unpacked tarball +native_dir=$(find "${TMP_DIR}" -maxdepth 1 -mindepth 1 -type d) + +cd "${native_dir}" || exit 1 + +# Make the native library +export USERFLAGS="$@" +make + +# Make sure it didn't fail +if [[ $? != 0 ]]; then + echo "Make failed!" + exit 1 +fi + +# "install" the artifact +cp libaccumulo.* "${final_native_target}" || exit 1 + +# Clean up our temp directory +rm -rf "${TMP_DIR}" + +echo "Successfully installed native library" http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/check-tservers ---------------------------------------------------------------------- diff --git a/assemble/scripts/check-tservers b/assemble/scripts/check-tservers new file mode 100755 index 0000000..7f9850e --- /dev/null +++ b/assemble/scripts/check-tservers @@ -0,0 +1,199 @@ +#! /usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script will check the configuration and uniformity of all the nodes in a cluster. +# Checks +# each node is reachable via ssh +# login identity is the same +# the physical memory is the same +# the mounts are the same on each machine +# a set of writable locations (typically different disks) are in fact writable +# +# In order to check for writable partitions, you must configure the WRITABLE variable below. +# + +import subprocess +import time +import select +import os +import sys +import fcntl +import signal +if not sys.platform.startswith('linux'): + sys.stderr.write('This script only works on linux, sorry.\n') + sys.exit(1) + +TIMEOUT = 5 +WRITABLE = [] +#WRITABLE = ['/srv/hdfs1', '/srv/hdfs2', '/srv/hdfs3'] + +def ssh(tserver, *args): + 'execute a command on a remote tserver and return the Popen handle' + handle = subprocess.Popen( ('ssh', '-o', 'StrictHostKeyChecking=no', '-q', '-A', '-n', tserver) + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + handle.tserver = tserver + handle.finished = False + handle.out = '' + return handle + +def wait(handles, seconds): + 'wait for lots of handles simultaneously, and kill anything that doesn\'t return in seconds time\n' + 'Note that stdout will be stored on the handle as the "out" field and "finished" will be set to True' + handles = handles[:] + stop = time.time() + seconds + for h in handles: + fcntl.fcntl(h.stdout, fcntl.F_SETFL, os.O_NONBLOCK) + while handles and time.time() < stop: + wait = min(0, stop - time.time()) + handleMap = dict( [(h.stdout, h) for h in handles] ) + rd, wr, err = select.select(handleMap.keys(), [], [], wait) + for r in rd: + handle = handleMap[r] + while 1: + more = handle.stdout.read(1024) + if more == '': + handles.remove(handle) + handle.poll() + handle.wait() + handle.finished = True + handle.out += more + if len(more) < 1024: + break + for handle in handles: + os.kill(handle.pid, signal.SIGKILL) + handle.poll() + +def runAll(tservers, *cmd): + 'Run the given command on all the tservers, returns Popen handles' + handles = [] + for tserver in tservers: + handles.append(ssh(tserver, *cmd)) + wait(handles, TIMEOUT) + return handles + +def checkIdentity(tservers): + 'Ensure the login identity is consistent across the tservers' + handles = runAll(tservers, 'id', '-u', '-n') + bad = set() + myIdentity = os.popen('id -u -n').read().strip() + for h in handles: + if not h.finished or h.returncode != 0: + print '#', 'cannot look at identity on', h.tserver + bad.add(h.tserver) + else: + identity = h.out.strip() + if identity != myIdentity: + print '#', h.tserver, 'inconsistent identity', identity + bad.add(h.tserver) + return bad + +def checkMemory(tservers): + 'Run free on all tservers and look for weird results' + handles = runAll(tservers, 'free') + bad = set() + mem = {} + swap = {} + for h in handles: + if not h.finished or h.returncode != 0: + print '#', 'cannot look at memory on', h.tserver + bad.add(h.tserver) + else: + if h.out.find('Swap:') < 0: + print '#',h.tserver,'has no swap' + bad.add(h.tserver) + continue + lines = h.out.split('\n') + for line in lines: + if line.startswith('Mem:'): + mem.setdefault(line.split()[1],set()).add(h.tserver) + if line.startswith('Swap:'): + swap.setdefault(line.split()[1],set()).add(h.tserver) + # order memory sizes by most common + mems = sorted([(len(v), k, v) for k, v in mem.items()], reverse=True) + mostCommon = float(mems[0][1]) + for _, size, tservers in mems[1:]: + fract = abs(mostCommon - float(size)) / mostCommon + if fract > 0.05: + print '#',', '.join(tservers), ': unusual memory size', size + bad.update(tservers) + swaps = sorted([(len(v), k, v) for k, v in swap.items()], reverse=True) + mostCommon = float(mems[0][1]) + for _, size, tservers in swaps[1:]: + fract = abs(mostCommon - float(size) / mostCommon) + if fract > 0.05: + print '#',', '.join(tservers), ': unusual swap size', size + bad.update(tservers) + return bad + +def checkWritable(tservers): + 'Touch all the directories that should be writable by this user return any nodes that fail' + if not WRITABLE: + print '# WRITABLE value not configured, not checking partitions' + return [] + handles = runAll(tservers, 'touch', *WRITABLE) + bad = set() + for h in handles: + if not h.finished or h.returncode != 0: + bad.add(h.tserver) + print '#', h.tserver, 'some drives are not writable' + return bad + +def checkMounts(tservers): + 'Check the file systems that are mounted and report any that are unusual' + handles = runAll(tservers, 'mount') + mounts = {} + finished = set() + bad = set() + for handle in handles: + if handle.finished and handle.returncode == 0: + for line in handle.out.split('\n'): + words = line.split() + if len(words) < 5: continue + if words[4] == 'nfs': continue + if words[0].find(':/') >= 0: continue + mount = words[2] + mounts.setdefault(mount, set()).add(handle.tserver) + finished.add(handle.tserver) + else: + bad.add(handle.tserver) + print '#', handle.tserver, 'did not finish' + for m in sorted(mounts.keys()): + diff = finished - mounts[m] + if diff: + bad.update(diff) + print '#', m, 'not mounted on', ', '.join(diff) + return bad + +def main(argv): + if len(argv) < 1: + sys.stderr.write('Usage: check_tservers tservers\n') + sys.exit(1) + sys.stdin.close() + tservers = set() + for tserver in open(argv[0]): + hashPos = tserver.find('#') + if hashPos >= 0: + tserver = tserver[:hashPos] + tserver = tserver.strip() + if not tserver: continue + tservers.add(tserver) + bad = set() + for test in checkIdentity, checkMemory, checkMounts, checkWritable: + bad.update(test(tservers - bad)) + for tserver in sorted(tservers - bad): + print tserver + +main(sys.argv[1:]) http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/config-server.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/config-server.sh b/assemble/scripts/config-server.sh new file mode 100755 index 0000000..a9ae645 --- /dev/null +++ b/assemble/scripts/config-server.sh @@ -0,0 +1,85 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Guarantees that Accumulo and its environment variables are set for start +# and stop scripts. Should always be run after config.sh. +# +# Parameters checked by script +# ACCUMULO_VERIFY_ONLY set to skip actions that would alter the local filesystem +# +# Values set by script that can be user provided. If not provided script attempts to infer. +# MONITOR Machine to run monitor daemon on. Used by start-here.sh script +# +# Iff ACCUMULO_VERIFY_ONLY is not set, this script will +# * Check for standalone mode (lack of masters and tservers files) +# - Do appropriate set up +# * Ensure the presense of local role files (masters, tservers, gc, tracers) +# +# Values always set by script. +# SSH Default ssh parameters used to start daemons +# + +unset MASTER1 +if [[ -f "$ACCUMULO_CONF_DIR/masters" ]]; then + MASTER1=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters" | head -1) +fi + +if [[ -z "${MONITOR}" ]] ; then + MONITOR=$MASTER1 + if [[ -f "$ACCUMULO_CONF_DIR/monitor" ]]; then + MONITOR=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/monitor" | head -1) + fi + if [[ -z "${MONITOR}" ]] ; then + echo "Could not infer a Monitor role. You need to either define the MONITOR env variable, define \"${ACCUMULO_CONF_DIR}/monitor\", or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." + exit 1 + fi +fi +if [[ ! -f "$ACCUMULO_CONF_DIR/tracers" && -z "${ACCUMULO_VERIFY_ONLY}" ]]; then + if [[ -z "${MASTER1}" ]] ; then + echo "Could not find a master node to use as a default for the tracer role. Either set up \"${ACCUMULO_CONF_DIR}/tracers\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." + exit 1 + else + echo "$MASTER1" > "$ACCUMULO_CONF_DIR/tracers" + fi + +fi + +if [[ ! -f "$ACCUMULO_CONF_DIR/gc" && -z "${ACCUMULO_VERIFY_ONLY}" ]]; then + if [[ -z "${MASTER1}" ]] ; then + echo "Could not infer a GC role. You need to either set up \"${ACCUMULO_CONF_DIR}/gc\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." + exit 1 + else + echo "$MASTER1" > "$ACCUMULO_CONF_DIR/gc" + fi +fi + +SSH='ssh -qnf -o ConnectTimeout=2' + +# ACCUMULO-1985 provide a way to use the scripts and still bind to all network interfaces +export ACCUMULO_MONITOR_BIND_ALL=${ACCUMULO_MONITOR_BIND_ALL:-"false"} + +if [[ -z "${ACCUMULO_PID_DIR}" ]]; then + export ACCUMULO_PID_DIR="${ACCUMULO_HOME}/run" +fi +[[ -z ${ACCUMULO_VERIFY_ONLY} ]] && mkdir -p "${ACCUMULO_PID_DIR}" 2>/dev/null + +if [[ -z "${ACCUMULO_IDENT_STRING}" ]]; then + export ACCUMULO_IDENT_STRING="$USER" +fi + +# The number of .out and .err files to retain +export ACCUMULO_NUM_OUT_FILES=${ACCUMULO_NUM_OUT_FILES:-5} http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/config.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/config.sh b/assemble/scripts/config.sh new file mode 100755 index 0000000..adc0e03 --- /dev/null +++ b/assemble/scripts/config.sh @@ -0,0 +1,172 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Guarantees that Accumulo and its environment variables are set. +# +# Parameters checked by script +# ACCUMULO_VERIFY_ONLY set to skip actions that would alter the local filesystem +# +# Values set by script that can be user provided. If not provided script attempts to infer. +# ACCUMULO_CONF_DIR Location where accumulo-env.sh, accumulo-site.xml and friends will be read from +# ACCUMULO_HOME Home directory for Accumulo +# ACCUMULO_LOG_DIR Directory for Accumulo daemon logs +# ACCUMULO_VERSION Accumulo version name +# HADOOP_PREFIX Prefix to the home dir for hadoop. +# +# Iff ACCUMULO_VERIFY_ONLY is not set, this script will +# * Ensure the existence of ACCUMULO_LOG_DIR on the current host +# +# Values always set by script. +# MALLOC_ARENA_MAX To work around a memory management bug (see ACCUMULO-847) +# HADOOP_HOME Home dir for hadoop. TODO fix this. +# +# Values set by script if certain files exist +# ACCUMULO_JAAS_CONF Location of jaas.conf file. Needed by JAAS for things like Kerberos based logins +# ACCUMULO_KRB5_CONF Location of krb5.conf file. Needed by Kerberos subsystems to find login servers +# +# NUMA related environment variables +# ACCUMULO_ENABLE_NUMACTL (Default: true) Use numactl if present on the system +# ACCUMULO_NUMACTL_OPTIONS (Default: --interleave=all) options to the numactl command + +if [ -z "${ACCUMULO_HOME}" ] ; then + # Start: Resolve Script Directory + SOURCE="${BASH_SOURCE[0]}" + while [[ -h $SOURCE ]]; do # resolve $SOURCE until the file is no longer a symlink + bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) + SOURCE=$(readlink "$SOURCE") + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located + done + bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) + script=$( basename "$SOURCE" ) + # Stop: Resolve Script Directory + + ACCUMULO_HOME=$( cd -P "${bin}"/.. && pwd ) + export ACCUMULO_HOME +fi + +if [[ ! -d ${ACCUMULO_HOME} ]]; then + echo "ACCUMULO_HOME=${ACCUMULO_HOME} is not a valid directory. Please make sure it exists" + exit 1 +fi + +export ACCUMULO_CONF_DIR="${ACCUMULO_CONF_DIR:-$ACCUMULO_HOME/conf}" +if [[ -z $ACCUMULO_CONF_DIR || ! -d $ACCUMULO_CONF_DIR ]] +then + echo "ACCUMULO_CONF_DIR=$ACCUMULO_CONF_DIR is not a valid directory. Please make sure it exists" + exit 1 +fi + +if [[ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ]] ; then + . "$ACCUMULO_CONF_DIR/accumulo-env.sh" +elif [[ -z $ACCUMULO_TEST ]] ; then + # + # Attempt to bootstrap configuration and continue + # + echo + echo 'Accumulo is not properly configured.' + echo + echo 'Try running $ACCUMULO_HOME/bin/bootstrap_config.sh and then editing' + echo '$ACCUMULO_HOME/conf/accumulo-env.sh' + echo + exit 1 +fi + +if [[ -f $ACCUMULO_CONF_DIR/slaves ]]; then + echo "ERROR: A 'slaves' file was found in $ACCUMULO_CONF_DIR/" + echo "Accumulo now reads tablet server hosts from 'tservers' and requires that the 'slaves' file not be present to reduce confusion." + echo "Please rename the 'slaves' file to 'tservers' or remove it if both exist." + exit 1 +fi + +if [[ ! -f $ACCUMULO_CONF_DIR/tservers ]]; then + echo "ERROR: A 'tservers' file was not found at $ACCUMULO_CONF_DIR/tservers" + echo "Please make sure it exists and is configured with tablet server hosts." + exit 1 +fi + +[[ -z $ACCUMULO_LOG_DIR ]] && ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs +[[ -z $ACCUMULO_VERIFY_ONLY ]] && mkdir -p $ACCUMULO_LOG_DIR 2>/dev/null + +export ACCUMULO_LOG_DIR + +if [[ -z "$HADOOP_PREFIX" ]] +then + HADOOP_PREFIX="$(which hadoop)" + if [[ -z "$HADOOP_PREFIX" ]] + then + echo "You must set HADOOP_PREFIX" + exit 1 + fi + HADOOP_PREFIX=$(dirname "$HADOOP_PREFIX") + HADOOP_PREFIX=$(dirname "$HADOOP_PREFIX") +fi +if [[ ! -d "$HADOOP_PREFIX" ]] +then + echo "HADOOP_PREFIX, which has a value of $HADOOP_PREFIX, is not a directory." + exit 1 +fi +export HADOOP_PREFIX + +ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} +ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} +NUMA=`which numactl 2>/dev/null` +NUMACTL_EXISTS="$?" +if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" +else + export NUMA_CMD="" +fi + +NUM_TSERVERS=${NUM_TSERVERS:-1} + +# Validate that NUM_TSERVERS is a positive integer +if ! [[ $NUM_TSERVERS =~ ^[0-9]+$ ]]; then + echo "NUM_TSERVERS, when defined in accumulo-env.sh, should be a positive number, is '$NUM_TSERVERS'" + exit 1 +fi + +# NUMA sanity checks +if [[ $NUM_TSERVERS -eq 1 && -n $TSERVER_NUMA_OPTIONS ]]; then + echo "TSERVER_NUMA_OPTIONS declared when NUM_TSERVERS is 1, use ACCUMULO_NUMACTL_OPTIONS instead" + exit 1 +fi +if [[ $NUM_TSERVERS -gt 1 && -n $TSERVER_NUMA_OPTIONS && ${#TSERVER_NUMA_OPTIONS[*]} -ne $NUM_TSERVERS ]]; then + echo "TSERVER_NUMA_OPTIONS is declared, but not the same size as NUM_TSERVERS" + exit 1 +fi + +export HADOOP_HOME=$HADOOP_PREFIX +export HADOOP_HOME_WARN_SUPPRESS=true + +# See HADOOP-7154 and ACCUMULO-847 +export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1} + +# Check for jaas.conf configuration +if [[ -z ${ACCUMULO_JAAS_CONF} ]]; then + if [[ -f ${ACCUMULO_CONF_DIR}/jaas.conf ]]; then + export ACCUMULO_JAAS_CONF=${ACCUMULO_CONF_DIR}/jaas.conf + fi +fi + +# Check for krb5.conf configuration +if [[ -z ${ACCUMULO_KRB5_CONF} ]]; then + if [[ -f ${ACCUMULO_CONF_DIR}/krb5.conf ]]; then + export ACCUMULO_KRB5_CONF=${ACCUMULO_CONF_DIR}/krb5.conf + fi +fi + + http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/generate-monitor-certificate.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/generate-monitor-certificate.sh b/assemble/scripts/generate-monitor-certificate.sh new file mode 100755 index 0000000..5e5745f --- /dev/null +++ b/assemble/scripts/generate-monitor-certificate.sh @@ -0,0 +1,84 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) + SOURCE=$(readlink "$SOURCE") + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) +# Stop: Resolve Script Directory + +. "$bin"/config.sh + +ALIAS="default" +KEYPASS=$(LC_CTYPE=C tr -dc '#-~' < /dev/urandom | tr -d '<>&' | head -c 20) +STOREPASS=$(LC_CTYPE=C tr -dc '#-~' < /dev/urandom | tr -d '<>&' | head -c 20) +KEYSTOREPATH="$ACCUMULO_HOME/conf/keystore.jks" +TRUSTSTOREPATH="$ACCUMULO_HOME/conf/cacerts.jks" +CERTPATH="$ACCUMULO_HOME/conf/server.cer" + +if [[ -e "$KEYSTOREPATH" ]]; then + rm -i "$KEYSTOREPATH" + if [[ -e "$KEYSTOREPATH" ]]; then + echo "KeyStore already exists, exiting" + exit 1 + fi +fi + +if [[ -e "$TRUSTSTOREPATH" ]]; then + rm -i "$TRUSTSTOREPATH" + if [[ -e "$TRUSTSTOREPATH" ]]; then + echo "TrustStore already exists, exiting" + exit 2 + fi +fi + +if [[ -e "$CERTPATH" ]]; then + rm -i "$CERTPATH" + if [[ -e "$CERTPATH" ]]; then + echo "Certificate already exists, exiting" + exit 3 + fi +fi + +"${JAVA_HOME}/bin/keytool" -genkey -alias "$ALIAS" -keyalg RSA -keypass "$KEYPASS" -storepass "$KEYPASS" -keystore "$KEYSTOREPATH" +"${JAVA_HOME}/bin/keytool" -export -alias "$ALIAS" -storepass "$KEYPASS" -file "$CERTPATH" -keystore "$KEYSTOREPATH" +"${JAVA_HOME}/bin/keytool" -import -v -trustcacerts -alias "$ALIAS" -file "$CERTPATH" -keystore "$TRUSTSTOREPATH" -storepass "$STOREPASS" <<< "yes" + +echo +echo "keystore and truststore generated. now add the following to accumulo-site.xml:" +echo +echo " <property>" +echo " <name>monitor.ssl.keyStore</name>" +echo " <value>$KEYSTOREPATH</value>" +echo " </property>" +echo " <property>" +echo " <name>monitor.ssl.keyStorePassword</name>" +echo " <value>$KEYPASS</value>" +echo " </property>" +echo " <property>" +echo " <name>monitor.ssl.trustStore</name>" +echo " <value>$TRUSTSTOREPATH</value>" +echo " </property>" +echo " <property>" +echo " <name>monitor.ssl.trustStorePassword</name>" +echo " <value>$STOREPASS</value>" +echo " </property>" +echo http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/log-forwarder.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/log-forwarder.sh b/assemble/scripts/log-forwarder.sh new file mode 100755 index 0000000..d625580 --- /dev/null +++ b/assemble/scripts/log-forwarder.sh @@ -0,0 +1,50 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This script executes a program that will forward some or all of the logs to a running instance of Chainsaw v2. +# To use this script, start Chainsaw on a host and create a new XMLSocketReceiver. This script +# accepts the following command line parameters +# +# host [required] - host running Chainsaw. Must be accessible via the network from this server +# port [required] - port that XMLSocketReceiver is listening on. +# filter [optional] - filter for log file names, * and ? are valid wildcards +# start [optional] - filter log messages beginning at this time (format is yyyyMMddHHmmss) +# end [optional] - filter log messages ending at this time (default is now, format is yyyyMMddHHmmss) +# level [optional] - filter log messages with this level and higher +# regex [optional] - filter log messages that match this regex (follows java.util.regex.Pattern syntax) +# +# +# Example: +# +# LogForwarder.sh -h 127.0.0.1 -p 4448 -f tserver* -s 2010010100001 -e 20100101235959 -l INFO -m .*scan.* +# + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) + SOURCE=$(readlink "$SOURCE") + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) +script=$( basename "$SOURCE" ) +# Stop: Resolve Script Directory + +. "$bin"/config.sh + +"${JAVA_HOME}/bin/java" -cp "$ACCUMULO_HOME/lib" org.apache.accumulo.server.util.SendLogToChainsaw -d "$ACCUMULO_LOG_DIR" "$@" http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/start-all.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/start-all.sh b/assemble/scripts/start-all.sh new file mode 100755 index 0000000..e2830aa --- /dev/null +++ b/assemble/scripts/start-all.sh @@ -0,0 +1,77 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +# Stop: Resolve Script Directory + +. "$bin"/config.sh +. "$bin"/config-server.sh +unset DISPLAY + +if [ ! -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then + echo "${ACCUMULO_CONF_DIR}/accumulo-env.sh does not exist. Please make sure you configure Accumulo before you run anything" + echo "We provide examples you can copy in ${ACCUMULO_HOME}/conf/examples/ which are set up for your memory footprint" + exit 1 +fi + +if [ -z "$ZOOKEEPER_HOME" ] ; then + echo "ZOOKEEPER_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh" + exit 1 +fi +if [ ! -d $ZOOKEEPER_HOME ]; then + echo "ZOOKEEPER_HOME is not a directory: $ZOOKEEPER_HOME" + echo "Please check the setting, either globally or in accumulo-env.sh." + exit 1 +fi + +ZOOKEEPER_VERSION=$(find -L $ZOOKEEPER_HOME -maxdepth 1 -name "zookeeper-[0-9]*.jar" | head -1) +if [ -z "$ZOOKEEPER_VERSION" ]; then + echo "A Zookeeper JAR was not found in $ZOOKEEPER_HOME." + echo "Please check ZOOKEEPER_HOME, either globally or in accumulo-env.sh." + exit 1 +fi +ZOOKEEPER_VERSION=$(basename "${ZOOKEEPER_VERSION##*-}" .jar) + +if [[ "$ZOOKEEPER_VERSION" < "3.4.0" ]]; then + echo "WARN : Using Zookeeper $ZOOKEEPER_VERSION. Use version 3.4.0 or greater. Older versions may not work reliably."; +fi + +${bin}/start-server.sh $MONITOR monitor + +if [ "$1" != "--notTservers" ]; then + ${bin}/tup.sh +fi + +${bin}/accumulo org.apache.accumulo.master.state.SetGoalState NORMAL +for master in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters"); do + ${bin}/start-server.sh $master master +done + +for gc in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/gc"); do + ${bin}/start-server.sh $gc gc +done + +for tracer in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/tracers"); do + ${bin}/start-server.sh $tracer tracer +done http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/start-daemon.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/start-daemon.sh b/assemble/scripts/start-daemon.sh new file mode 100755 index 0000000..9d6404f --- /dev/null +++ b/assemble/scripts/start-daemon.sh @@ -0,0 +1,167 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +script=$( basename "$SOURCE" ) +# Stop: Resolve Script Directory + +usage="Usage: start-daemon.sh <host> <service>" + +rotate_log () { + logfile=$1; + max_retained=$2; + if [[ ! $max_retained =~ ^[0-9]+$ ]] || [[ $max_retained -lt 1 ]] ; then + echo "ACCUMULO_NUM_OUT_FILES should be a positive number, but was '$max_retained'" + exit 1 + fi + + if [ -f "$logfile" ]; then # rotate logs + while [ $max_retained -gt 1 ]; do + prev=`expr $max_retained - 1` + [ -f "$logfile.$prev" ] && mv -f "$logfile.$prev" "$logfile.$max_retained" + max_retained=$prev + done + mv -f "$logfile" "$logfile.$max_retained"; + fi +} + +if [[ $# -ne 2 ]]; then + echo $usage + exit 2 +fi + +. "$bin"/config.sh +. "$bin"/config-server.sh + +HOST="$1" +ADDRESS=$HOST +host "$1" >/dev/null 2>&1 +if [[ $? != 0 ]]; then + LOGHOST=$HOST +else + LOGHOST=$(host "$HOST" | head -1 | cut -d' ' -f1) +fi +SERVICE=$2 + +TSERVERS=$(wc -l < "${ACCUMULO_CONF_DIR}/tservers") + +# When the hostname provided is the alias/shortname, try to use the FQDN to make +# sure we send the right address to the Accumulo process. +if [[ "$HOST" = "$(hostname -s)" ]]; then + HOST="$(hostname -f)" + ADDRESS="$HOST" +fi + +# ACCUMULO-1985 Allow monitor to bind on all interfaces +if [[ ${SERVICE} == "monitor" && ${ACCUMULO_MONITOR_BIND_ALL} == "true" ]]; then + ADDRESS="0.0.0.0" +fi + +COMMAND="${bin}/accumulo" +if [ "${ACCUMULO_WATCHER}" = "true" ]; then + COMMAND="${bin}/accumulo_watcher.sh ${LOGHOST}" +fi + +OUTFILE="${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out" +ERRFILE="${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err" + +# Rotate the .out and .err files +rotate_log "$OUTFILE" ${ACCUMULO_NUM_OUT_FILES} +rotate_log "$ERRFILE" ${ACCUMULO_NUM_OUT_FILES} + +if [[ "$SERVICE" != "tserver" || $NUM_TSERVERS -eq 1 ]]; then + # Check the pid file to figure out if its already running. + PID_FILE="${ACCUMULO_PID_DIR}/accumulo-${ACCUMULO_IDENT_STRING}-${SERVICE}.pid" + if [ -f ${PID_FILE} ]; then + PID=`cat ${PID_FILE}` + if kill -0 $PID 2>/dev/null; then + # Starting an already-started service shouldn't be an error per LSB + echo "$HOST : $SERVICE already running (${PID})" + exit 0 + fi + fi + echo "Starting $SERVICE on $HOST" + + # Fork the process, store the pid + nohup ${NUMA_CMD} "$COMMAND" "${SERVICE}" --address "${ADDRESS}" >"$OUTFILE" 2>"$ERRFILE" < /dev/null & + echo $! > ${PID_FILE} + +else + + S="$SERVICE" + for (( t=1; t<=$NUM_TSERVERS; t++)); do + + SERVICE="$S-$t" + + # Check the pid file to figure out if its already running. + PID_FILE="${ACCUMULO_PID_DIR}/accumulo-${ACCUMULO_IDENT_STRING}-${SERVICE}.pid" + if [ -f ${PID_FILE} ]; then + PID=`cat ${PID_FILE}` + if kill -0 $PID 2>/dev/null; then + # Starting an already-started service shouldn't be an error per LSB + echo "$HOST : $SERVICE already running (${PID})" + continue + fi + fi + echo "Starting $SERVICE on $HOST" + + ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} + ACCUMULO_NUMACTL_OPTIONS=${TSERVER_NUMA_OPTIONS[$t]} + if [[ "$ACCUMULO_ENABLE_NUMACTL" == "true" ]]; then + NUMA=`which numactl 2>/dev/null` + NUMACTL_EXISTS=$? + if [[ ( ${NUMACTL_EXISTS} -eq 0 ) ]]; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" + else + export NUMA_CMD="" + fi + fi + + # We want the files to be consistently named with the log files + # server_identifier_hostname.{out,err}, e.g. tserver_2_fqdn.out + OUTFILE="${ACCUMULO_LOG_DIR}/${S}_${t}_${LOGHOST}.out" + ERRFILE="${ACCUMULO_LOG_DIR}/${S}_${t}_${LOGHOST}.err" + + # Rotate the .out and .err files + rotate_log "$OUTFILE" ${ACCUMULO_NUM_OUT_FILES} + rotate_log "$ERRFILE" ${ACCUMULO_NUM_OUT_FILES} + + # Fork the process, store the pid + nohup ${NUMA_CMD} "$COMMAND" "${SERVICE}" --address "${ADDRESS}" >"$OUTFILE" 2>"$ERRFILE" < /dev/null & + echo $! > ${PID_FILE} + + done + +fi + +# Check the max open files limit and selectively warn +MAX_FILES_OPEN=$(ulimit -n) + +if [[ -n $MAX_FILES_OPEN && -n $TSERVERS ]] ; then + MAX_FILES_RECOMMENDED=${MAX_FILES_RECOMMENDED:-32768} + if (( TSERVERS > 10 )) && (( MAX_FILES_OPEN < MAX_FILES_RECOMMENDED )) + then + echo "WARN : Max open files on $HOST is $MAX_FILES_OPEN, recommend $MAX_FILES_RECOMMENDED" >&2 + fi +fi http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/start-here.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/start-here.sh b/assemble/scripts/start-here.sh new file mode 100755 index 0000000..ccba80f --- /dev/null +++ b/assemble/scripts/start-here.sh @@ -0,0 +1,78 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This script starts all the accumulo services on this host +# + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +# Stop: Resolve Script Directory + +. "$bin"/config.sh +. "$bin"/config-server.sh + +IFCONFIG=/sbin/ifconfig +[[ ! -x $IFCONFIG ]] && IFCONFIG='/bin/netstat -ie' + +IP=$($IFCONFIG 2>/dev/null| grep "inet[^6]" | awk '{print $2}' | sed 's/addr://' | grep -v 0.0.0.0 | grep -v 127.0.0.1 | head -n 1) +if [[ $? != 0 ]]; then + IP=$(python -c 'import socket as s; print s.gethostbyname(s.getfqdn())') +fi + +HOSTS="$(hostname -a 2> /dev/null) $(hostname) localhost 127.0.0.1 $IP" +for host in $HOSTS; do + if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/tservers"; then + "${bin}/start-server.sh" "$host" tserver + break + fi +done + +for host in $HOSTS; do + if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/masters"; then + "${bin}/accumulo" org.apache.accumulo.master.state.SetGoalState NORMAL + "${bin}/start-server.sh" "$host" master + break + fi +done + +for host in $HOSTS; do + if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/gc"; then + "${bin}/start-server.sh" "$host" gc + break + fi +done + +for host in $HOSTS; do + if [[ $host == "$MONITOR" ]]; then + "${bin}/start-server.sh" "$MONITOR" monitor + break + fi +done + +for host in $HOSTS; do + if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/tracers"; then + "${bin}/start-server.sh" "$host" tracer + break + fi +done http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/start-server.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/start-server.sh b/assemble/scripts/start-server.sh new file mode 100755 index 0000000..d5b7594 --- /dev/null +++ b/assemble/scripts/start-server.sh @@ -0,0 +1,57 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +script=$( basename "$SOURCE" ) +# Stop: Resolve Script Directory + +# Really, we still support the third <long_name> argument, but let's not tell people that.. +usage="Usage: start-server.sh <host> <service>" + +# Support the 3-arg invocation for backwards-compat +if [[ $# -ne 2 ]] && [[ $# -ne 3 ]]; then + echo $usage + exit 2 +fi + +. "$bin"/config.sh +. "$bin"/config-server.sh + +HOST="$1" +SERVICE="$2" + +IFCONFIG=/sbin/ifconfig +[[ ! -x $IFCONFIG ]] && IFCONFIG='/bin/netstat -ie' + +IP=$($IFCONFIG 2>/dev/null| grep "inet[^6]" | awk '{print $2}' | sed 's/addr://' | grep -v 0.0.0.0 | grep -v 127.0.0.1 | head -n 1) +if [[ $? != 0 ]] ; then + IP=$(python -c 'import socket as s; print s.gethostbyname(s.getfqdn())') +fi + +if [[ $HOST == "localhost" || $HOST == $(hostname -f) || $HOST == $(hostname -s) || $HOST == $IP ]]; then + "$bin/start-daemon.sh" "$HOST" "$SERVICE" +else + # Ensure that the provided configuration directory is sent with the command + echo $($SSH $HOST "bash -c 'ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR} $bin/start-daemon.sh \"$HOST\" \"$SERVICE\"'") +fi http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/stop-all.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/stop-all.sh b/assemble/scripts/stop-all.sh new file mode 100755 index 0000000..2825ef6 --- /dev/null +++ b/assemble/scripts/stop-all.sh @@ -0,0 +1,69 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [[ -h $SOURCE ]]; do # resolve $SOURCE until the file is no longer a symlink + bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) + SOURCE=$(readlink "$SOURCE") + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) +# Stop: Resolve Script Directory + +. "$bin"/config.sh +. "$bin"/config-server.sh + +echo "Stopping accumulo services..." +${bin}/accumulo admin "$@" stopAll + +if [[ $? != 0 ]]; then + echo "Invalid password or unable to connect to the master" + echo "Initiating forced shutdown in 15 seconds (Ctrl-C to abort)" + sleep 10 + echo "Initiating forced shutdown in 5 seconds (Ctrl-C to abort)" +else + echo "Accumulo shut down cleanly" + echo "Utilities and unresponsive servers will shut down in 5 seconds (Ctrl-C to abort)" +fi + +sleep 5 + +#look for master and gc processes not killed by 'admin stopAll' +for signal in TERM KILL ; do + for master in $(grep -v '^#' "$ACCUMULO_CONF_DIR/masters"); do + "${bin}/stop-server.sh" "$master" "$ACCUMULO_HOME/lib/accumulo-start.*.jar" master $signal + done + + for gc in $(grep -v '^#' "$ACCUMULO_CONF_DIR/gc"); do + "${bin}/stop-server.sh" "$gc" "$ACCUMULO_HOME/lib/accumulo-start.*.jar" gc $signal + done + + "${bin}/stop-server.sh" "$MONITOR" "$ACCUMULO_HOME/.*/accumulo-start.*.jar" monitor $signal + + for tracer in $(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/tracers"); do + "${bin}/stop-server.sh" "$tracer" "$ACCUMULO_HOME/.*/accumulo-start.*.jar" tracer $signal + done +done + +# stop tserver still running +"${bin}/tdown.sh" + +echo "Cleaning all server entries in ZooKeeper" +"$ACCUMULO_HOME/bin/accumulo" org.apache.accumulo.server.util.ZooZap -master -tservers -tracers --site-file "$ACCUMULO_CONF_DIR/accumulo-site.xml" + http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/stop-here.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/stop-here.sh b/assemble/scripts/stop-here.sh new file mode 100755 index 0000000..1edfb5b --- /dev/null +++ b/assemble/scripts/stop-here.sh @@ -0,0 +1,53 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This script safely stops all the accumulo services on this host +# + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +# Stop: Resolve Script Directory + +. "$bin"/config.sh + +# Determine hostname without errors to user +HOSTS_TO_CHECK=($(hostname -a 2> /dev/null | head -1) $(hostname -f)) + +if egrep -q localhost\|127.0.0.1 "$ACCUMULO_CONF_DIR/tservers"; then + "$bin/accumulo" admin stop localhost +else + for host in "${HOSTS_TO_CHECK[@]}"; do + if grep -q "$host" "$ACCUMULO_CONF_DIR"/tservers; then + "${bin}/accumulo" admin stop "$host" + fi + done +fi + +for HOSTNAME in "${HOSTS_TO_CHECK[@]}"; do + for signal in TERM KILL; do + for svc in tserver gc master monitor tracer; do + "$ACCUMULO_HOME"/bin/stop-server.sh "$HOSTNAME" "$ACCUMULO_HOME/lib/accumulo-start.jar" $svc $signal + done + done +done http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/stop-server.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/stop-server.sh b/assemble/scripts/stop-server.sh new file mode 100755 index 0000000..bba0f1e --- /dev/null +++ b/assemble/scripts/stop-server.sh @@ -0,0 +1,59 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) + SOURCE=$(readlink "$SOURCE") + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) +# Stop: Resolve Script Directory + +. "$bin"/config.sh +. "$bin"/config-server.sh + +HOST=$1 + +IFCONFIG=/sbin/ifconfig +[[ ! -x $IFCONFIG ]] && IFCONFIG='/bin/netstat -ie' + +IP=$($IFCONFIG 2>/dev/null| grep "inet[^6]" | awk '{print $2}' | sed 's/addr://' | grep -v 0.0.0.0 | grep -v 127.0.0.1 | head -n 1) +if [[ $? != 0 ]] +then + IP=$(python -c 'import socket as s; print s.gethostbyname(s.getfqdn())') +fi + +# only stop if there's not one already running +if [[ $HOST == localhost || $HOST = "$(hostname -s)" || $HOST = "$(hostname -f)" || $HOST = "$IP" ]] ; then + for PID_FILE in ${ACCUMULO_PID_DIR}/accumulo-${ACCUMULO_IDENT_STRING}-${3}*.pid; do + if [[ -f ${PID_FILE} ]]; then + echo "Stopping $3 on $1"; + kill -s "$4" `cat ${PID_FILE}` 2>/dev/null + rm -f ${PID_FILE} 2>/dev/null + fi; + done +else + for PID_FILE in $(ssh -q -o 'ConnectTimeout 8' "$1" ls "${ACCUMULO_PID_DIR}/accumulo-${ACCUMULO_IDENT_STRING}-${3}*.pid" 2>/dev/null); do + PID=$(ssh -q -o 'ConnectTimeout 8' "$1" cat "${PID_FILE}" 2>/dev/null) + if [[ ! -z $PID ]]; then + echo "Stopping $3 on $1"; + ssh -q -o 'ConnectTimeout 8' "$1" "kill -s $4 $PID 2>/dev/null; rm -f ${PID_FILE} 2>/dev/null" + fi + done +fi http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/tdown.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/tdown.sh b/assemble/scripts/tdown.sh new file mode 100755 index 0000000..dd81ddd --- /dev/null +++ b/assemble/scripts/tdown.sh @@ -0,0 +1,49 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +# Stop: Resolve Script Directory + +. "$bin"/config.sh + +HADOOP_CMD=$HADOOP_PREFIX/bin/hadoop +TSERVERS=$ACCUMULO_CONF_DIR/tservers +TSERVER_HOSTS=$(egrep -v '(^#|^\s*$)' "${TSERVERS}") + +echo "Stopping unresponsive tablet servers (if any)..." +for server in ${TSERVER_HOSTS}; do + # only start if there's not one already running + $ACCUMULO_HOME/bin/stop-server.sh $server "$ACCUMULO_HOME/lib/accumulo-start.jar" tserver TERM & +done + +sleep 10 + +echo "Stopping unresponsive tablet servers hard (if any)..." +for server in ${TSERVER_HOSTS}; do + # only start if there's not one already running + $ACCUMULO_HOME/bin/stop-server.sh $server "$ACCUMULO_HOME/lib/accumulo-start.jar" tserver KILL & +done + +echo "Cleaning tablet server entries from zookeeper" +$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.util.ZooZap -tservers http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/templates/accumulo-env.sh ---------------------------------------------------------------------- diff --git a/assemble/scripts/templates/accumulo-env.sh b/assemble/scripts/templates/accumulo-env.sh new file mode 100644 index 0000000..a91b4b3 --- /dev/null +++ b/assemble/scripts/templates/accumulo-env.sh @@ -0,0 +1,89 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### +### Configure these environment variables to point to your local installations. +### +### The functional tests require conditional values, so keep this style: +### +### test -z "$JAVA_HOME" && export JAVA_HOME=/usr/lib/jvm/java +### +### +### Note that the -Xmx -Xms settings below require substantial free memory: +### you may want to use smaller values, especially when running everything +### on a single machine. +### +if [[ -z $HADOOP_HOME ]] ; then + test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop +else + HADOOP_PREFIX="$HADOOP_HOME" + unset HADOOP_HOME +fi + +# hadoop-2.0: +test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop" + +test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java +test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper +test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs +test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${tServerHigh_tServerLow} " +test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${masterHigh_masterLow}" +test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${monitorHigh_monitorLow}" +test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="${gcHigh_gcLow}" +test -z "$ACCUMULO_SHELL_OPTS" && export ACCUMULO_SHELL_OPTS="${shellHigh_shellLow}" +test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true -XX:+CMSClassUnloadingEnabled" +test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="${otherHigh_otherLow}" +test -z "${ACCUMULO_PID_DIR}" && export ACCUMULO_PID_DIR="${ACCUMULO_HOME}/run" +# what do when the JVM runs out of heap memory +export ACCUMULO_KILL_CMD='kill -9 %p' + +### Optionally look for hadoop and accumulo native libraries for your +### platform in additional directories. (Use DYLD_LIBRARY_PATH on Mac OS X.) +### May not be necessary for Hadoop 2.x or using an RPM that installs to +### the correct system library directory. +# export LD_LIBRARY_PATH=${HADOOP_PREFIX}/lib/native/${PLATFORM}:${LD_LIBRARY_PATH} + +# Should the monitor bind to all network interfaces -- default: false +# export ACCUMULO_MONITOR_BIND_ALL="true" + +# Should process be automatically restarted +# export ACCUMULO_WATCHER="true" + +# What settings should we use for the watcher, if enabled +export UNEXPECTED_TIMESPAN="3600" +export UNEXPECTED_RETRIES="2" + +export OOM_TIMESPAN="3600" +export OOM_RETRIES="5" + +export ZKLOCK_TIMESPAN="600" +export ZKLOCK_RETRIES="5" + +# The number of .out and .err files per process to retain +# export ACCUMULO_NUM_OUT_FILES=5 + +export NUM_TSERVERS=1 + +### Example for configuring multiple tservers per host. Note that the ACCUMULO_NUMACTL_OPTIONS +### environment variable is used when NUM_TSERVERS is 1 to preserve backwards compatibility. +### If NUM_TSERVERS is greater than 2, then the TSERVER_NUMA_OPTIONS array is used if defined. +### If TSERVER_NUMA_OPTIONS is declared but not the correct size, then the service will not start. +### +### export NUM_TSERVERS=2 +### declare -a TSERVER_NUMA_OPTIONS +### TSERVER_NUMA_OPTIONS[1]="--cpunodebind 0" +### TSERVER_NUMA_OPTIONS[2]="--cpunodebind 1" http://git-wip-us.apache.org/repos/asf/accumulo/blob/24ba5568/assemble/scripts/templates/accumulo-site.xml ---------------------------------------------------------------------- diff --git a/assemble/scripts/templates/accumulo-site.xml b/assemble/scripts/templates/accumulo-site.xml new file mode 100644 index 0000000..ff490b1 --- /dev/null +++ b/assemble/scripts/templates/accumulo-site.xml @@ -0,0 +1,176 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + <!-- Put your site-specific accumulo configurations here. The available configuration values along with their defaults are documented in docs/config.html Unless + you are simply testing at your workstation, you will most definitely need to change the three entries below. --> + + <property> + <name>instance.volumes</name> + <value></value> + <description>comma separated list of URIs for volumes. example: hdfs://localhost:9000/accumulo</description> + </property> + + <property> + <name>instance.zookeeper.host</name> + <value>localhost:2181</value> + <description>comma separated list of zookeeper servers</description> + </property> + + <property> + <name>instance.secret</name> + <value>DEFAULT</value> + <description>A secret unique to a given instance that all servers must know in order to communicate with one another. + Change it before initialization. To + change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd], + and then update this file. + </description> + </property> + + <property> + <name>tserver.memory.maps.max</name> + <value>${memMapMax}</value> + </property> + + <property> + <name>tserver.memory.maps.native.enabled</name> + <value>${nativeEnabled}</value> + </property> + + <property> + <name>tserver.cache.data.size</name> + <value>${cacheDataSize}</value> + </property> + + <property> + <name>tserver.cache.index.size</name> + <value>${cacheIndexSize}</value> + </property> + + <property> + <name>trace.token.property.password</name> + <!-- change this to the root user's password, and/or change the user below --> + <value>secret</value> + </property> + + <!-- Kerberos requirements --> + <property> + <name>instance.rpc.sasl.enabled</name> + <value>true</value> + </property> + + <property> + <name>general.kerberos.keytab</name> + <value>${keytab}</value> + </property> + + <property> + <name>general.kerberos.principal</name> + <value>${principal}</value> + </property> + + <property> + <name>trace.token.type</name> + <value>org.apache.accumulo.core.client.security.tokens.KerberosToken</value> + </property> + + <property> + <name>instance.security.authenticator</name> + <value>org.apache.accumulo.server.security.handler.KerberosAuthenticator</value> + </property> + + <property> + <name>instance.security.authorizor</name> + <value>org.apache.accumulo.server.security.handler.KerberosAuthorizor</value> + </property> + + <property> + <name>instance.security.permissionHandler</name> + <value>org.apache.accumulo.server.security.handler.KerberosPermissionHandler</value> + </property> + <!-- End Kerberos requirements --> + + <property> + <name>trace.user</name> + <value>${traceUser}</value> + </property> + + <property> + <name>tserver.sort.buffer.size</name> + <value>${sortBufferSize}</value> + </property> + + <property> + <name>tserver.walog.max.size</name> + <value>${waLogMaxSize}</value> + </property> +${mvnProjBaseDir} + <property> + <name>general.classpaths</name> + + <value> + <!-- Accumulo requirements --> + $ACCUMULO_HOME/lib/accumulo-server.jar, + $ACCUMULO_HOME/lib/accumulo-core.jar, + $ACCUMULO_HOME/lib/accumulo-start.jar, + $ACCUMULO_HOME/lib/accumulo-fate.jar, + $ACCUMULO_HOME/lib/accumulo-proxy.jar, + $ACCUMULO_HOME/lib/[^.].*.jar, + <!-- ZooKeeper requirements --> + $ZOOKEEPER_HOME/zookeeper[^.].*.jar, + <!-- Common Hadoop requirements --> + $HADOOP_CONF_DIR, + <!-- Hadoop 2 requirements --> + $HADOOP_PREFIX/share/hadoop/common/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/common/lib/(?!slf4j)[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/hdfs/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/mapreduce/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/yarn/[^.].*.jar, + $HADOOP_PREFIX/share/hadoop/yarn/lib/jersey.*.jar, + <!-- End Hadoop 2 requirements --> + <!-- HDP 2.0 requirements --> + /usr/lib/hadoop/[^.].*.jar, + /usr/lib/hadoop/lib/[^.].*.jar, + /usr/lib/hadoop-hdfs/[^.].*.jar, + /usr/lib/hadoop-mapreduce/[^.].*.jar, + /usr/lib/hadoop-yarn/[^.].*.jar, + /usr/lib/hadoop-yarn/lib/jersey.*.jar, + <!-- End HDP 2.0 requirements --> + <!-- HDP 2.2 requirements --> + /usr/hdp/current/hadoop-client/[^.].*.jar, + /usr/hdp/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, + /usr/hdp/current/hadoop-hdfs-client/[^.].*.jar, + /usr/hdp/current/hadoop-mapreduce-client/[^.].*.jar, + /usr/hdp/current/hadoop-yarn-client/[^.].*.jar, + /usr/hdp/current/hadoop-yarn-client/lib/jersey.*.jar, + /usr/hdp/current/hive-client/lib/hive-accumulo-handler.jar + <!-- End HDP 2.2 requirements --> + <!-- IOP 4.1 requirements --> + /usr/iop/current/hadoop-client/[^.].*.jar, + /usr/iop/current/hadoop-client/lib/(?!slf4j)[^.].*.jar, + /usr/iop/current/hadoop-hdfs-client/[^.].*.jar, + /usr/iop/current/hadoop-mapreduce-client/[^.].*.jar, + /usr/iop/current/hadoop-yarn-client/[^.].*.jar, + /usr/iop/current/hadoop-yarn-client/lib/jersey.*.jar, + /usr/iop/current/hive-client/lib/hive-accumulo-handler.jar + <!-- End IOP 4.1 requirements --> + </value> + <description>Classpaths that accumulo checks for updates and class files.</description> + </property> +</configuration>
