Repository: incubator-distributedlog Updated Branches: refs/heads/master 2a677d15a -> a546e6207
Update a couple of scripts Mostly style. Author: Franck Cuny <fc...@apache.org> Author: Leigh Stewart <lstew...@twitter.com> Author: Khurrum Nasim <khurrumnas...@gmail.com> Reviewers: Sijie Guo <si...@apache.org> Closes #46 from franckcuny/fcuny/update-scripts and squashes the following commits: 50c16ec [Franck Cuny] Rename a variable in one of the shell script. 924ba7e [Franck Cuny] Run `git rev-parse` only once when building the bundle. f1d3b3f [Franck Cuny] Remove more duplicated shell scripts. 90ce66f [Franck Cuny] Do not duplicate code for the 'bundle' script. 13987ea [Franck Cuny] Build cobertura report only on master for successful builds 525ec9a [Khurrum Nasim] Fix deadlock on BKSyncLogReaderDLSN 23c9190 [Leigh Stewart] Add documentation for distributedlog-benchmark 85aa8a7 [Franck Cuny] Make the shell style more consistent. 4f81ba2 [Franck Cuny] Rely on git to find the root directory. Project: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/commit/a546e620 Tree: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/tree/a546e620 Diff: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/diff/a546e620 Branch: refs/heads/master Commit: a546e62076ff34d48c2cbf6a4b803d789e4b5466 Parents: 2a677d1 Author: Franck Cuny <fc...@apache.org> Authored: Fri Dec 2 19:53:39 2016 -0800 Committer: Sijie Guo <si...@apache.org> Committed: Fri Dec 2 19:53:39 2016 -0800 ---------------------------------------------------------------------- distributedlog-benchmark/bin/bundle | 17 +-- distributedlog-benchmark/bin/dbench | 134 ++++------------------- distributedlog-core/bin/dlog | 160 ++++++--------------------- distributedlog-service/bin/bundle | 17 +-- distributedlog-service/bin/dlog | 179 ++++++++----------------------- scripts/bundle | 45 ++++++++ scripts/common.sh | 126 ++++++++++++++++++++++ scripts/runner | 148 +++++-------------------- scripts/snapshot | 18 ++-- 9 files changed, 305 insertions(+), 539 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/a546e620/distributedlog-benchmark/bin/bundle ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/bin/bundle b/distributedlog-benchmark/bin/bundle index 353f438..838f74d 100755 --- a/distributedlog-benchmark/bin/bundle +++ b/distributedlog-benchmark/bin/bundle @@ -18,19 +18,8 @@ # * limitations under the License. # */ -BASEDIR=$(dirname "$0") -DISTRIBUTEDLOG_ROOT="${BASEDIR}/../.." - set -e -shopt -s dotglob -cd ${DISTRIBUTEDLOG_ROOT} -rm -rf distributedlog-benchmark/lib -rm -rf dist/distributedlog-benchmark* -mkdir -p dist/distributedlog-benchmark/{conf,lib} -mvn clean install -Ptwitter-ostrich-provider -pl distributedlog-benchmark -am -DskipTests -cp -rf distributedlog-benchmark/conf/* dist/distributedlog-benchmark/conf -cp -rf distributedlog-benchmark/lib/* dist/distributedlog-benchmark/lib -cp -rf distributedlog-benchmark/target/distributedlog-benchmark-*.jar dist/distributedlog-benchmark -cd dist -zip -r distributedlog-benchmark.zip distributedlog-benchmark >/dev/null +DLOG_ROOT=$(git rev-parse --show-toplevel) + +bash "${DLOG_ROOT}"/scripts/bundle benchmark http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/a546e620/distributedlog-benchmark/bin/dbench ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/bin/dbench b/distributedlog-benchmark/bin/dbench index b84e133..42dd5b0 100755 --- a/distributedlog-benchmark/bin/dbench +++ b/distributedlog-benchmark/bin/dbench @@ -18,59 +18,13 @@ # * limitations under the License. # */ -# check if net.ipv6.bindv6only is set to 1 -bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) -if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] -then - echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" - echo "For more info (the following page also applies to dlog): http://wiki.apache.org/hadoop/HadoopIPv6"; - exit 1 -fi +set -e -# See the following page for extensive details on setting -# up the JVM to accept JMX remote management: -# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# by default we allow local JMX connections -if [ "x$JMXLOCALONLY" = "x" ] -then - JMXLOCALONLY=false -fi +DLOG_ROOT=$(git rev-parse --show-toplevel) -if [ "x$JMXDISABLE" = "x" ] -then - echo "JMX enabled by default" >&2 - # for some reason these two options are necessary on jdk6 on Ubuntu - # accord to the docs they are not necessary, but otw jconsole cannot - # do a local attach - JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" -else - echo "JMX disabled by user request" >&2 -fi +DLOG_HOME="${DLOG_ROOT}/distributedlog-benchmark" -BINDIR=`dirname "$0"` -DLOG_HOME=`cd $BINDIR/.. > /dev/null;pwd` - -DEFAULT_LOG_CONF=$DLOG_HOME/conf/log4j.properties - -source $DLOG_HOME/conf/dlogenv.sh - -# exclude tests jar -RELEASE_JAR=`ls $DLOG_HOME/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1` -if [ $? == 0 ]; then - DLOG_JAR=$RELEASE_JAR -fi - -# exclude tests jar -BUILT_JAR=`ls $DLOG_HOME/target/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1` -if [ $? != 0 ] && [ ! -e "$DLOG_JAR" ]; then - echo "\nCouldn't find dlog jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; -elif [ -e "$BUILT_JAR" ]; then - DLOG_JAR=$BUILT_JAR -fi - -dbench_help() { +usage() { cat <<EOF Usage: dlog <command> where command is one of: @@ -89,67 +43,13 @@ These variable can also be set in conf/dlogenv.sh EOF } -add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${DLOG_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - fi - DLOG_CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -if [ -d "$DLOG_HOME/lib" ]; then - for i in $DLOG_HOME/lib/*.jar; do - DLOG_CLASSPATH=$DLOG_CLASSPATH:$i - done -else - add_maven_deps_to_classpath -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - dbench_help; - exit 1; -fi +cd "${DLOG_ROOT}" +source ./scripts/common.sh # get arguments COMMAND=$1 shift -if [ -z "$DLOG_LOG_CONF" ]; then - DLOG_LOG_CONF=$DEFAULT_LOG_CONF -fi - -DLOG_CLASSPATH="$DLOG_JAR:$DLOG_CLASSPATH:$DLOG_EXTRA_CLASSPATH" -if [ "$DLOG_LOG_CONF" != "" ]; then - DLOG_CLASSPATH="`dirname $DLOG_LOG_CONF`:$DLOG_CLASSPATH" - OPTS="$OPTS -Dlog4j.configuration=`basename $DLOG_LOG_CONF`" -fi -OPTS="-cp $DLOG_CLASSPATH $OPTS $DLOG_EXTRA_OPTS" - -OPTS="$OPTS $DLOG_EXTRA_OPTS" - -# Disable ipv6 as it can cause issues -OPTS="$OPTS -Djava.net.preferIPv4Stack=true" - -# log directory & file -DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-"INFO,R"} -DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DLOG_HOME/logs"} -DLOG_LOG_FILE=${DLOG_LOG_FILE:-"dbench.log"} - -#Configure log configuration system properties -OPTS="$OPTS -Ddlog.root.logger=$DLOG_ROOT_LOGGER" -OPTS="$OPTS -Ddlog.log.dir=$DLOG_LOG_DIR" -OPTS="$OPTS -Ddlog.log.file=$DLOG_LOG_FILE" - BENCH_ARGS=""" --provider ${STATS_PROVIDER} \\ --conf ${BENCH_CONF_FILE:-"${DLOG_HOME}/conf/benchmark.conf"} \\ @@ -165,7 +65,9 @@ BENCH_ARGS=""" #Change to DLOG_HOME to support relative paths cd "$DLOG_HOME" -if [ $COMMAND == "bkwrite" ]; then + +case "${COMMAND}" in + bkwrite) BENCH_WRITE_ARGS=""" --messagesize ${MSG_SIZE} \\ --rate ${INITIAL_RATE} \\ @@ -175,7 +77,8 @@ if [ $COMMAND == "bkwrite" ]; then """ BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_WRITE_ARGS} \\ --mode dlwrite \\" exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@ -elif [ $COMMAND == "write" ]; then + ;; + write) BENCH_WRITE_ARGS=""" --messagesize ${MSG_SIZE} \\ --rate ${INITIAL_RATE} \\ @@ -185,7 +88,8 @@ elif [ $COMMAND == "write" ]; then """ BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_WRITE_ARGS} \\ --mode write \\" exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@ -elif [ $COMMAND == "read" ]; then + ;; + read) BENCH_READ_ARGS=""" --readers-per-stream ${NUM_READERS_PER_STREAM} \\ --max-stream-id ${MAX_STREAM_ID} \\ @@ -193,10 +97,12 @@ elif [ $COMMAND == "read" ]; then """ BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_READ_ARGS} \\ --mode read \\" exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@ -elif [ $COMMAND == "help" ]; then - dbench_help; -else + ;; + help) + usage + ;; + *) exec java $OPTS $COMMAND $@ -fi - + ;; +esac http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/a546e620/distributedlog-core/bin/dlog ---------------------------------------------------------------------- diff --git a/distributedlog-core/bin/dlog b/distributedlog-core/bin/dlog index 169d5d9..c4f9ed2 100755 --- a/distributedlog-core/bin/dlog +++ b/distributedlog-core/bin/dlog @@ -18,154 +18,58 @@ # * limitations under the License. # */ -# check if net.ipv6.bindv6only is set to 1 -bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) -if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] -then - echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" - echo "For more info (the following page also applies to dlog): http://wiki.apache.org/hadoop/HadoopIPv6"; - exit 1 -fi +set -e -# See the following page for extensive details on setting -# up the JVM to accept JMX remote management: -# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# by default we allow local JMX connections -if [ "x$JMXLOCALONLY" = "x" ] -then - JMXLOCALONLY=false -fi +DLOG_ROOT=$(git rev-parse --show-toplevel) +DLOG_HOME="${DLOG_ROOT}" -if [ "x$JMXDISABLE" = "x" ] -then - echo "JMX enabled by default" >&2 - # for some reason these two options are necessary on jdk6 on Ubuntu - # accord to the docs they are not necessary, but otw jconsole cannot - # do a local attach - JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" -else - echo "JMX disabled by user request" >&2 -fi - -BINDIR=`dirname "$0"` -DLOG_HOME=`cd $BINDIR/.. > /dev/null;pwd` - -DEFAULT_LOG_CONF=$DLOG_HOME/conf/log4j.properties - -source $DLOG_HOME/conf/dlogenv.sh - -# exclude tests jar -RELEASE_JAR=`ls $DLOG_HOME/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1` -if [ $? == 0 ]; then - DLOG_JAR=$RELEASE_JAR -fi - -# exclude tests jar -BUILT_JAR=`ls $DLOG_HOME/target/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1` -if [ $? != 0 ] && [ ! -e "$DLOG_JAR" ]; then - echo "\nCouldn't find dlog jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; -elif [ -e "$BUILT_JAR" ]; then - DLOG_JAR=$BUILT_JAR -fi - -dlog_help() { - cat <<EOF -Usage: dlog <command> +usage() { + cat <<EOF +Usage: runner <command> where command is one of: - local Run distributedlog sandbox - example Run distributedlog example - tool Run distributedlog tool - proxy_tool Run distributedlog proxy tool to interact with proxies - balancer Run distributedlog balancer - admin Run distributedlog admin tool - help This help message + local Run distributedlog sandbox + example Run distributedlog example + tool Run distributedlog tool + proxy_tool Run distributedlog proxy tool to interact with proxies + balancer Run distributedlog balancer + admin Run distributedlog admin tool + help This help message or command is the full name of a class with a defined main() method. Environment variables: - DLOG_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) - DLOG_EXTRA_OPTS Extra options to be passed to the jvm - DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath + DLOG_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) + DLOG_EXTRA_OPTS Extra options to be passed to the jvm + DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath These variable can also be set in conf/dlogenv.sh EOF } -add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${DLOG_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - fi - DLOG_CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -if [ -d "$DLOG_HOME/lib" ]; then - for i in $DLOG_HOME/lib/*.jar; do - DLOG_CLASSPATH=$DLOG_CLASSPATH:$i - done -else - add_maven_deps_to_classpath -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - dlog_help; - exit 1; -fi +cd "${DLOG_ROOT}" +source ./scripts/common.sh # get arguments COMMAND=$1 shift -if [ -z "$DLOG_LOG_CONF" ]; then - DLOG_LOG_CONF=$DEFAULT_LOG_CONF -fi - -DLOG_CLASSPATH="$DLOG_JAR:$DLOG_CLASSPATH:$DLOG_EXTRA_CLASSPATH" -if [ "$DLOG_LOG_CONF" != "" ]; then - DLOG_CLASSPATH="`dirname $DLOG_LOG_CONF`:$DLOG_CLASSPATH" - OPTS="$OPTS -Dlog4j.configuration=`basename $DLOG_LOG_CONF`" -fi -OPTS="-cp $DLOG_CLASSPATH $OPTS $DLOG_EXTRA_OPTS" - -OPTS="$OPTS $DLOG_EXTRA_OPTS" - -# Disable ipv6 as it can cause issues -OPTS="$OPTS -Djava.net.preferIPv4Stack=true" - -# log directory & file -DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-"INFO,R"} -DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DLOG_HOME/logs"} -DLOG_LOG_FILE=${DLOG_LOG_FILE:-"dlog.log"} - -#Configure log configuration system properties -OPTS="$OPTS -Ddlog.root.logger=$DLOG_ROOT_LOGGER" -OPTS="$OPTS -Ddlog.log.dir=$DLOG_LOG_DIR" -OPTS="$OPTS -Ddlog.log.file=$DLOG_LOG_FILE" - #Change to DLOG_HOME to support relative paths cd "$DLOG_HOME" -if [ $COMMAND == "local" ]; then + +case "${COMMAND}" in + local) exec java $OPTS $JMX_ARGS com.twitter.distributedlog.LocalDLMEmulator $@ -elif [ $COMMAND == "tool" ]; then + ;; + tool) exec java $OPTS com.twitter.distributedlog.tools.Tool com.twitter.distributedlog.tools.DistributedLogTool $@ -elif [ $COMMAND == "admin" ]; then + ;; + admin) exec java $OPTS com.twitter.distributedlog.tools.Tool com.twitter.distributedlog.admin.DistributedLogAdmin $@ -elif [ $COMMAND == "help" ]; then - dlog_help; -else + ;; + help) + usage + ;; + *) exec java $OPTS $COMMAND $@ -fi - - + ;; +esac http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/a546e620/distributedlog-service/bin/bundle ---------------------------------------------------------------------- diff --git a/distributedlog-service/bin/bundle b/distributedlog-service/bin/bundle index 3ad94fb..bd5b1a0 100755 --- a/distributedlog-service/bin/bundle +++ b/distributedlog-service/bin/bundle @@ -18,19 +18,8 @@ # * limitations under the License. # */ -BASEDIR=$(dirname "$0") -DISTRIBUTEDLOG_ROOT="${BASEDIR}/../.." - set -e -shopt -s dotglob -cd ${DISTRIBUTEDLOG_ROOT} -rm -rf distributedlog-service/lib -rm -rf dist/distributedlog-service* -mkdir -p dist/distributedlog-service/{conf,lib} -mvn clean install -pl distributedlog-service -am -DskipTests -cp -r distributedlog-service/conf/* dist/distributedlog-service/conf -cp -r distributedlog-service/lib/* dist/distributedlog-service/lib -cp distributedlog-service/target/distributedlog-service-*.jar dist/distributedlog-service -cd dist -zip -r distributedlog-service.zip distributedlog-service >/dev/null +DLOG_ROOT=$(git rev-parse --show-toplevel) + +bash "${DLOG_ROOT}"/scripts/bundle service http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/a546e620/distributedlog-service/bin/dlog ---------------------------------------------------------------------- diff --git a/distributedlog-service/bin/dlog b/distributedlog-service/bin/dlog index 07bc047..b9a01e1 100755 --- a/distributedlog-service/bin/dlog +++ b/distributedlog-service/bin/dlog @@ -18,171 +18,76 @@ # * limitations under the License. # */ -# check if net.ipv6.bindv6only is set to 1 -bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) -if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] -then - echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" - echo "For more info (the following page also applies to dlog): http://wiki.apache.org/hadoop/HadoopIPv6"; - exit 1 -fi +set -e -# See the following page for extensive details on setting -# up the JVM to accept JMX remote management: -# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# by default we allow local JMX connections -if [ "x$JMXLOCALONLY" = "x" ] -then - JMXLOCALONLY=false -fi +DLOG_ROOT=$(git rev-parse --show-toplevel) -if [ "x$JMXDISABLE" = "x" ] -then - echo "JMX enabled by default" >&2 - # for some reason these two options are necessary on jdk6 on Ubuntu - # accord to the docs they are not necessary, but otw jconsole cannot - # do a local attach - JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" -else - echo "JMX disabled by user request" >&2 -fi +DLOG_HOME="${DLOG_ROOT}" -BINDIR=`dirname "$0"` -DLOG_HOME=`cd $BINDIR/.. > /dev/null;pwd` - -DEFAULT_BK_CONF=$DLOG_HOME/conf/bookie.conf -DEFAULT_LOG_CONF=$DLOG_HOME/conf/log4j.properties - -source $DLOG_HOME/conf/dlogenv.sh - -# exclude tests jar -RELEASE_JAR=`ls $DLOG_HOME/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1` -if [ $? == 0 ]; then - DLOG_JAR=$RELEASE_JAR -fi - -# exclude tests jar -BUILT_JAR=`ls $DLOG_HOME/target/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1` -if [ $? != 0 ] && [ ! -e "$DLOG_JAR" ]; then - echo "\nCouldn't find dlog jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; -elif [ -e "$BUILT_JAR" ]; then - DLOG_JAR=$BUILT_JAR -fi - -dlog_help() { - cat <<EOF -Usage: dlog <command> +usage() { + cat <<EOF +Usage: runner <command> where command is one of: - local Run distributedlog sandbox - example Run distributedlog example - tool Run distributedlog tool - proxy_tool Run distributedlog proxy tool to interact with proxies - balancer Run distributedlog balancer - admin Run distributedlog admin tool - zkshell Run zookeeper shell - bkshell Run bookkeeper shell - help This help message + local Run distributedlog sandbox + example Run distributedlog example + tool Run distributedlog tool + proxy_tool Run distributedlog proxy tool to interact with proxies + balancer Run distributedlog balancer + admin Run distributedlog admin tool + zkshell Run zookeeper shell + bkshell Run bookkeeper shell + help This help message or command is the full name of a class with a defined main() method. Environment variables: - DLOG_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) - DLOG_EXTRA_OPTS Extra options to be passed to the jvm - DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath - BOOKIE_CONF Bookie Configuration file (default: $DEFAULT_BK_CONF) + DLOG_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) + DLOG_EXTRA_OPTS Extra options to be passed to the jvm + DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath These variable can also be set in conf/dlogenv.sh EOF } -add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${DLOG_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - fi - DLOG_CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -if [ -d "$DLOG_HOME/lib" ]; then - for i in $DLOG_HOME/lib/*.jar; do - DLOG_CLASSPATH=$DLOG_CLASSPATH:$i - done -else - add_maven_deps_to_classpath -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - dlog_help; - exit 1; -fi +cd "${DLOG_ROOT}" +source ./scripts/common.sh # get arguments COMMAND=$1 shift -if [ -z "$DLOG_LOG_CONF" ]; then - DLOG_LOG_CONF=$DEFAULT_LOG_CONF -fi - -if [ -z "$BOOKIE_CONF" ]; then - BOOKIE_CONF=$DEFAULT_BK_CONF -fi - -DLOG_CLASSPATH="$DLOG_JAR:$DLOG_CLASSPATH:$DLOG_EXTRA_CLASSPATH" -if [ "$DLOG_LOG_CONF" != "" ]; then - DLOG_CLASSPATH="`dirname $DLOG_LOG_CONF`:$DLOG_CLASSPATH" - OPTS="$OPTS -Dlog4j.configuration=`basename $DLOG_LOG_CONF`" -fi -OPTS="-cp $DLOG_CLASSPATH $OPTS $DLOG_EXTRA_OPTS" - -OPTS="$OPTS $DLOG_EXTRA_OPTS" - -# Disable ipv6 as it can cause issues -OPTS="$OPTS -Djava.net.preferIPv4Stack=true" - -# log directory & file -DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-"INFO,R"} -DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DLOG_HOME/logs"} -DLOG_LOG_FILE=${DLOG_LOG_FILE:-"dlog.log"} - -#Configure log configuration system properties -OPTS="$OPTS -Ddlog.root.logger=$DLOG_ROOT_LOGGER" -OPTS="$OPTS -Ddlog.log.dir=$DLOG_LOG_DIR" -OPTS="$OPTS -Ddlog.log.file=$DLOG_LOG_FILE" - #Change to DLOG_HOME to support relative paths cd "$DLOG_HOME" -if [ $COMMAND == "local" ]; then + +case "${COMMAND}" in + local) exec java $OPTS $JMX_ARGS com.twitter.distributedlog.LocalDLMEmulator $@ -elif [ $COMMAND == "tool" ]; then + ;; + tool) exec java $OPTS com.twitter.distributedlog.tools.Tool com.twitter.distributedlog.tools.DistributedLogTool $@ -elif [ $COMMAND == "proxy_tool" ]; then + ;; + proxy_tool) exec java $OPTS com.twitter.distributedlog.tools.Tool com.twitter.distributedlog.service.tools.ProxyTool $@ -elif [ $COMMAND == "balancer" ]; then + ;; + balancer) exec java $OPTS com.twitter.distributedlog.tools.Tool com.twitter.distributedlog.service.balancer.BalancerTool $@ -elif [ $COMMAND == "admin" ]; then + ;; + admin) exec java $OPTS com.twitter.distributedlog.tools.Tool com.twitter.distributedlog.admin.DistributedLogAdmin $@ -elif [ $COMMAND == "zkshell" ]; then + ;; + zkshell) exec java $OPTS org.apache.zookeeper.ZooKeeperMain -server $@ -elif [ $COMMAND == "bkshell" ]; then + ;; + bkshell) ENTRY_FORMATTER_ARG="-DentryFormatterClass=${ENTRY_FORMATTER_CLASS:-org.apache.bookkeeper.util.StringEntryFormatter}" exec java $OPTS $ENTRY_FORMATTER_ARG org.apache.bookkeeper.bookie.BookieShell -conf $BOOKIE_CONF $@ -elif [ $COMMAND == "help" ]; then - dlog_help; -else + ;; + help) + usage + ;; + *) exec java $OPTS $COMMAND $@ -fi + ;; +esac http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/a546e620/scripts/bundle ---------------------------------------------------------------------- diff --git a/scripts/bundle b/scripts/bundle new file mode 100755 index 0000000..a0c8a3a --- /dev/null +++ b/scripts/bundle @@ -0,0 +1,45 @@ +#!/bin/bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +set -e +shopt -s dotglob + +COMPONENT=$1 +if [[ -z "${COMPONENT}" ]]; then + echo "You need to specify the component" 2>&1 + exit 1 +fi + +cd "${DLOG_ROOT}" + +rm -rf "distributedlog-${COMPONENT}/lib" +rm -rf "dist/distributedlog-${COMPONENT}*" + +mkdir -p "dist/distributedlog-${COMPONENT}"/{conf,lib} + +mvn install -Ptwitter-ostrich-provider -pl "distributedlog-${COMPONENT}" -am -DskipTests + +cp -rf distributedlog-"${COMPONENT}"/conf/* "dist/distributedlog-${COMPONENT}/conf/" +cp -rf "distributedlog-${COMPONENT}"/lib/* "dist/distributedlog-${COMPONENT}/lib" +cp -rf "distributedlog-${COMPONENT}"/target/distributedlog-"${COMPONENT}"-*.jar "dist/distributedlog-${COMPONENT}" + +cd dist + +zip -r "distributedlog-${COMPONENT}.zip" "distributedlog-${COMPONENT}" >/dev/null http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/a546e620/scripts/common.sh ---------------------------------------------------------------------- diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100755 index 0000000..433f93b --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +if [ $(uname) == "Linux" ]; then + # check if net.ipv6.bindv6only is set to 1 + bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) + if [ -n "${bindv6only}" ] && [ "${bindv6only}" -eq "1" ]; then + echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" + echo "For more info (the following page also applies to DistributedLog): http://wiki.apache.org/hadoop/HadoopIPv6"; + exit 1 + fi +fi + +# See the following page for extensive details on setting +# up the JVM to accept JMX remote management: +# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# by default we allow local JMX connections +if [ -z "${JMXLOCALONLY}" ]; then + JMXLOCALONLY=false +fi + +if [ -z "${JMXDISABLE}" ]; then + echo "JMX enabled by default" >&2 + # for some reason these two options are necessary on jdk6 on Ubuntu + # accord to the docs they are not necessary, but otw jconsole cannot + # do a local attach + JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=${JMXLOCALONLY}" +else + echo "JMX disabled by user request" >&2 +fi + +DEFAULT_LOG_CONF="${DLOG_HOME}/conf/log4j.properties" + +[ -f "${DLOG_HOME}/conf/dlogenv.sh" ] && source "${DLOG_HOME}/conf/dlogenv.sh" + +# exclude tests jar +RELEASE_JAR=$(ls "${DLOG_HOME}/distributedlog-*.jar" 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1) +if [ $? == 0 ]; then + DLOG_JAR="${RELEASE_JAR}" +fi + +# exclude tests jar +BUILT_JAR=$(ls "${DLOG_HOME}/target/distributedlog-*.jar" 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1) + +if [ -e "${BUILD_JAR}" ] && [ -e "${DLOG_JAR}" ]; then + echo "\nCouldn't find dlog jar."; + echo "Make sure you've run 'mvn package'\n"; + exit 1; +elif [ -e "${BUILT_JAR}" ]; then + DLOG_JAR="${BUILT_JAR}" +fi + +add_maven_deps_to_classpath() { + MVN="mvn" + if [ -n "${MAVEN_HOME}" ]; then + MVN="${MAVEN_HOME}/bin/mvn" + fi + + # Need to generate classpath from maven pom. This is costly so generate it + # and cache it. Save the file into our target dir so a mvn clean will get + # clean it up and force us create a new one. + f="${DLOG_HOME}/target/cached_classpath.txt" + if [ ! -f "${f}" ]; then + "${MVN}" -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + fi + DLOG_CLASSPATH="${CLASSPATH}":$(cat "${f}") +} + +if [ -d "${DLOG_HOME}/lib" ]; then + for i in ${DLOG_HOME}/lib/*.jar; do + DLOG_CLASSPATH="${DLOG_CLASSPATH}:${i}" + done +else + add_maven_deps_to_classpath +fi + +# if no args specified, show usage +if [ $# = 0 ]; then + usage + exit 1 +fi + +if [ -z "${DLOG_LOG_CONF}" ]; then + DLOG_LOG_CONF="${DEFAULT_LOG_CONF}" +fi + +DLOG_CLASSPATH="${DLOG_JAR}:${DLOG_CLASSPATH}:${DLOG_EXTRA_CLASSPATH}" +if [ -n "${DLOG_LOG_CONF}" ]; then + DLOG_CLASSPATH="$(dirname ${DLOG_LOG_CONF}):${DLOG_CLASSPATH}" + OPTS="${OPTS} -Dlog4j.configuration=$(basename ${DLOG_LOG_CONF})" +fi +OPTS="-cp ${DLOG_CLASSPATH} ${OPTS} ${DLOG_EXTRA_OPTS}" + +OPTS="${OPTS} ${DLOG_EXTRA_OPTS}" + +# Disable ipv6 as it can cause issues +OPTS="${OPTS} -Djava.net.preferIPv4Stack=true" + +# log directory & file +DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-"INFO,R"} +DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DLOG_HOME/logs"} +DLOG_LOG_FILE=${DLOG_LOG_FILE:-"dlog.log"} + +#Configure log configuration system properties +OPTS="$OPTS -Ddlog.root.logger=${DLOG_ROOT_LOGGER}" +OPTS="$OPTS -Ddlog.log.dir=${DLOG_LOG_DIR}" +OPTS="$OPTS -Ddlog.log.file=${DLOG_LOG_FILE}" http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/a546e620/scripts/runner ---------------------------------------------------------------------- diff --git a/scripts/runner b/scripts/runner index 7780f1a..cc05946 100755 --- a/scripts/runner +++ b/scripts/runner @@ -20,143 +20,49 @@ # * limitations under the License. # */ -# check if net.ipv6.bindv6only is set to 1 -bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) -if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] -then - echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" - echo "For more info (the following page also applies to dlog): http://wiki.apache.org/hadoop/HadoopIPv6"; - exit 1 -fi +set -e -# See the following page for extensive details on setting -# up the JVM to accept JMX remote management: -# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# by default we allow local JMX connections -if [ "x$JMXLOCALONLY" = "x" ] -then - JMXLOCALONLY=false -fi +DLOG_ROOT=$(git rev-parse --show-toplevel) -if [ "x$JMXDISABLE" = "x" ] -then - echo "JMX enabled by default" >&2 - # for some reason these two options are necessary on jdk6 on Ubuntu - # accord to the docs they are not necessary, but otw jconsole cannot - # do a local attach - JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" -else - echo "JMX disabled by user request" >&2 -fi +DLOG_HOME="${DLOG_ROOT}/distributedlog-service" -BINDIR=`dirname "$0"` -DLOG_HOME=`cd $BINDIR/.. > /dev/null;pwd` - -DEFAULT_LOG_CONF=$DLOG_HOME/conf/log4j.properties - -source $DLOG_HOME/conf/dlogenv.sh - -# exclude tests jar -RELEASE_JAR=`ls $DLOG_HOME/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1` -if [ $? == 0 ]; then - DLOG_JAR=$RELEASE_JAR -fi - -# exclude tests jar -BUILT_JAR=`ls $DLOG_HOME/target/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1` -if [ $? != 0 ] && [ ! -e "$DLOG_JAR" ]; then - echo "\nCouldn't find dlog jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; -elif [ -e "$BUILT_JAR" ]; then - DLOG_JAR=$BUILT_JAR -fi - -dlog_help() { - cat <<EOF +usage() { + cat <<EOF Usage: runner <command> where command is one of: - run Run distributedlog example - help This help message + run Run distributedlog example + help This help message or command is the full name of a class with a defined main() method. Environment variables: - DLOG_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) - DLOG_EXTRA_OPTS Extra options to be passed to the jvm - DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath + DLOG_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) + DLOG_EXTRA_OPTS Extra options to be passed to the jvm + DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath These variable can also be set in conf/dlogenv.sh EOF } -add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${DLOG_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - fi - DLOG_CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -if [ -d "$DLOG_HOME/lib" ]; then - for i in $DLOG_HOME/lib/*.jar; do - DLOG_CLASSPATH=$DLOG_CLASSPATH:$i - done -else - add_maven_deps_to_classpath -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - dlog_help; - exit 1; -fi +cd "${DLOG_ROOT}" +source ./scripts/common.sh # get arguments -COMMAND=$1 +COMMAND="${1}" shift -if [ -z "$DLOG_LOG_CONF" ]; then - DLOG_LOG_CONF=$DEFAULT_LOG_CONF -fi - -DLOG_CLASSPATH="$DLOG_JAR:$DLOG_CLASSPATH:$DLOG_EXTRA_CLASSPATH" -if [ "$DLOG_LOG_CONF" != "" ]; then - DLOG_CLASSPATH="`dirname $DLOG_LOG_CONF`:$DLOG_CLASSPATH" - OPTS="$OPTS -Dlog4j.configuration=`basename $DLOG_LOG_CONF`" -fi -OPTS="-cp $DLOG_CLASSPATH $OPTS $DLOG_EXTRA_OPTS" - -OPTS="$OPTS $DLOG_EXTRA_OPTS" - -# Disable ipv6 as it can cause issues -OPTS="$OPTS -Djava.net.preferIPv4Stack=true" - -# log directory & file -DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-"INFO,R"} -DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DLOG_HOME/logs"} -DLOG_LOG_FILE=${DLOG_LOG_FILE:-"dlog.log"} - -#Configure log configuration system properties -OPTS="$OPTS -Ddlog.root.logger=$DLOG_ROOT_LOGGER" -OPTS="$OPTS -Ddlog.log.dir=$DLOG_LOG_DIR" -OPTS="$OPTS -Ddlog.log.file=$DLOG_LOG_FILE" - #Change to DLOG_HOME to support relative paths -cd "$DLOG_HOME" -if [ $COMMAND == "run" ]; then - exec java $OPTS $@ -elif [ $COMMAND == "help" ]; then - dlog_help; -else - dlog_help; -fi +cd "${DLOG_HOME}" + +case "${COMMAND}" in + run) + java ${OPTS} $@ + ;; + help) + usage + ;; + *) + usage + exit 1 + ;; +esac http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/a546e620/scripts/snapshot ---------------------------------------------------------------------- diff --git a/scripts/snapshot b/scripts/snapshot index 081f307..2c9e285 100755 --- a/scripts/snapshot +++ b/scripts/snapshot @@ -20,15 +20,11 @@ # * limitations under the License. # */ -BASEDIR=$(dirname "$0") -DISTRIBUTEDLOG_ROOT="${BASEDIR}/.." - set -e shopt -s dotglob -cd ${DISTRIBUTEDLOG_ROOT} - -gitsha=`git rev-parse HEAD` +DISTRIBUTEDLOG_ROOT=$(git rev-parse --show-toplevel) +GITSHA=$(git rev-parse HEAD) # clean the dist find . -name lib | xargs rm -r @@ -36,7 +32,7 @@ rm -rf dist/* # create the layout for module in distributedlog-service distributedlog-benchmark distributedlog-tutorials/distributedlog-basic distributedlog-tutorials/distributedlog-messaging; do - mkdir -p dist/release/${module}/{bin,conf,lib} + mkdir -p "dist/release/${module}/{bin,conf,lib}" done # build the package @@ -69,7 +65,7 @@ cp -rf distributedlog-tutorials/distributedlog-messaging/target/distributedlog-m # release cd dist/release -zip -r distributedlog-all-${gitsha}.zip distributedlog-service distributedlog-benchmark distributedlog-tutorials > /dev/null -zip -r distributedlog-service-${gitsha}.zip distributedlog-service >/dev/null -zip -r distributedlog-benchmark-${gitsha}.zip distributedlog-benchmark >/dev/null -zip -r distributedlog-tutorials-${gitsha}.zip distributedlog-tutorials >/dev/null +zip -r "distributedlog-all-${GITSHA}.zip" distributedlog-service distributedlog-benchmark distributedlog-tutorials > /dev/null +zip -r "distributedlog-service-${GITSHA}.zip" distributedlog-service >/dev/null +zip -r "distributedlog-benchmark-${GITSHA}.zip" distributedlog-benchmark >/dev/null +zip -r "distributedlog-tutorials-${GITSHA}.zip" distributedlog-tutorials >/dev/null
