Repository: incubator-distributedlog Updated Branches: refs/heads/master fd63b34bc -> 927f0ca01
Clean up and fix a few shell scripts This is still in progress. - remove trailing white spaces in a few places - fix some scripts (not rely on git); style consistency - validate that the vagrant configuration works - one of the Dockerfile was not correct Author: Franck Cuny <[email protected]> Reviewers: Sijie Guo <[email protected]> Closes #55 from franckcuny/fcuny/hack-week-cleanup and squashes the following commits: fe3965d [Franck Cuny] Fix dlog script for core. f15cefd [Franck Cuny] fix a few more things in the common script. c32dee2 [Franck Cuny] Remove useless cd 622d3ab [Franck Cuny] Fix bundle for service. 93e9485 [Franck Cuny] A few more fixes for benchmarks' scripts. a3fc4ef [Franck Cuny] Remove trailing white spaces in the doc 27f90f1 [Franck Cuny] Rename bookkeeper hosts from node to bk. 21d89f8 [Franck Cuny] Do not assume git is available in the shell scripts. 480bc34 [Franck Cuny] Remove comment from Dockerfile a7ac7c0 [Franck Cuny] Git ignore pid files 179eddc [Franck Cuny] Ignore .vagrant directory created by vagrant. 9bf33fc [Franck Cuny] Create a valid snapshot. 5800829 [Franck Cuny] Remove Dockerfile and docker directory. Project: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/commit/927f0ca0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/tree/927f0ca0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/diff/927f0ca0 Branch: refs/heads/master Commit: 927f0ca0159a6fad9cf9a5618e313e2f033e598b Parents: fd63b34 Author: Franck Cuny <[email protected]> Authored: Thu Dec 15 16:20:57 2016 -0800 Committer: Franck Cuny <[email protected]> Committed: Thu Dec 15 16:20:57 2016 -0800 ---------------------------------------------------------------------- .gitignore | 4 + Dockerfile | 10 +- Vagrantfile | 5 +- distributedlog-benchmark/bin/bundle | 7 +- distributedlog-benchmark/bin/dbench | 15 ++- distributedlog-benchmark/conf/dlogenv.sh | 2 +- distributedlog-core/bin/dlog | 9 +- distributedlog-service/bin/bundle | 7 +- distributedlog-service/bin/dlog | 10 +- distributedlog-service/bin/dlog-daemon.sh | 170 ++++++++++++------------- docker/Dockerfile | 35 ----- docs/admin_guide/vagrant.rst | 4 +- docs/deployment/cluster.rst | 58 +++++---- docs/deployment/docker.rst | 2 +- scripts/bundle | 2 - scripts/common.sh | 6 +- scripts/runner | 3 +- scripts/snapshot | 11 +- vagrant/base.sh | 43 ++++--- vagrant/bk.sh | 64 +++++----- vagrant/zk.sh | 5 +- 21 files changed, 222 insertions(+), 250 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 3b719cd..914de41 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,7 @@ lib/ *.#* dist/ logs/ +*.pid + +# vagrant +.vagrant http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/Dockerfile ---------------------------------------------------------------------- diff --git a/Dockerfile b/Dockerfile index d0c90ae..f45e355 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,13 +24,9 @@ COPY . /opt/distributedlog-trunk/ ENV BROKER_ID 0 ENV ZK_SERVERS zk1:2181 -EXPOSE 3181 -EXPOSE 9001 -EXPOSE 4181 +EXPOSE 3181 +EXPOSE 9001 +EXPOSE 4181 EXPOSE 20001 -#CMD ["/bin/bash", "-c", "/opt/distributedlog-trunk/dist/release/vagrant/bk.sh", "$BROKER_ID", "127.0.0.1", "$ZK_SERVERS" ] ENTRYPOINT [ "/opt/distributedlog-trunk/dist/release/vagrant/bk_docker_wrapper.sh" ] - - - http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/Vagrantfile ---------------------------------------------------------------------- diff --git a/Vagrantfile b/Vagrantfile index 1aa733e..ca46785 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -34,7 +34,6 @@ if File.exists?(local_config_file) then eval(File.read(local_config_file), binding, "Vagrantfile.local") end -# TODO(ksweeney): RAM requirements are not empirical and can probably be significantly lowered. Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.hostmanager.enabled = enable_hostmanager config.hostmanager.manage_host = enable_dns @@ -92,7 +91,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| } (1..num_hosts).each { |i| - name = "node" + i.to_s + name = "bk" + i.to_s config.vm.define name do |broker| name_node(broker, name) ip_address = "192.168.50." + (50 + i).to_s @@ -106,6 +105,4 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| broker.vm.provision "shell", path: "vagrant/bk.sh", :args => [i.to_s, enable_dns ? name : ip_address, zookeeper_connect] end } - - end http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-benchmark/bin/bundle ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/bin/bundle b/distributedlog-benchmark/bin/bundle index 838f74d..c0fdcd8 100755 --- a/distributedlog-benchmark/bin/bundle +++ b/distributedlog-benchmark/bin/bundle @@ -20,6 +20,9 @@ set -e -DLOG_ROOT=$(git rev-parse --show-toplevel) +BASEDIR=$(dirname "$0") +DLOG_ROOT="${BASEDIR}/../.." -bash "${DLOG_ROOT}"/scripts/bundle benchmark +cd "${DLOG_ROOT}" + +bash scripts/bundle benchmark http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-benchmark/bin/dbench ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/bin/dbench b/distributedlog-benchmark/bin/dbench index 42dd5b0..ef58477 100755 --- a/distributedlog-benchmark/bin/dbench +++ b/distributedlog-benchmark/bin/dbench @@ -20,7 +20,8 @@ set -e -DLOG_ROOT=$(git rev-parse --show-toplevel) +BASEDIR=$(dirname "$0") +DLOG_ROOT="${BASEDIR}/../.." DLOG_HOME="${DLOG_ROOT}/distributedlog-benchmark" @@ -43,8 +44,7 @@ These variable can also be set in conf/dlogenv.sh EOF } -cd "${DLOG_ROOT}" -source ./scripts/common.sh +source ${DLOG_ROOT}/scripts/common.sh # get arguments COMMAND=$1 @@ -64,7 +64,7 @@ BENCH_ARGS=""" """ #Change to DLOG_HOME to support relative paths -cd "$DLOG_HOME" +#cd "$DLOG_HOME" case "${COMMAND}" in bkwrite) @@ -73,7 +73,7 @@ case "${COMMAND}" in --rate ${INITIAL_RATE} \\ --max-rate ${MAX_RATE} \\ --change-rate ${CHANGE_RATE} \\ - --change-interval ${CHANGE_RATE_INTERVAL} + --change-interval ${CHANGE_RATE_INTERVAL} \\ """ BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_WRITE_ARGS} \\ --mode dlwrite \\" exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@ @@ -85,6 +85,7 @@ case "${COMMAND}" in --max-rate ${MAX_RATE} \\ --change-rate ${CHANGE_RATE} \\ --change-interval ${CHANGE_RATE_INTERVAL} \\ + --finagle-name inet!localhost:8000 \\ """ BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_WRITE_ARGS} \\ --mode write \\" exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@ @@ -94,6 +95,7 @@ case "${COMMAND}" in --readers-per-stream ${NUM_READERS_PER_STREAM} \\ --max-stream-id ${MAX_STREAM_ID} \\ --truncation-interval ${TRUNCATION_INTERVAL} \\ + --finagle-name inet!localhost:8000 \\ """ BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_READ_ARGS} \\ --mode read \\" exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@ @@ -102,7 +104,8 @@ case "${COMMAND}" in usage ;; *) - exec java $OPTS $COMMAND $@ + set -x + echo java $OPTS $COMMAND $@ ;; esac http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-benchmark/conf/dlogenv.sh ---------------------------------------------------------------------- diff --git a/distributedlog-benchmark/conf/dlogenv.sh b/distributedlog-benchmark/conf/dlogenv.sh index 205606b..11abe9f 100644 --- a/distributedlog-benchmark/conf/dlogenv.sh +++ b/distributedlog-benchmark/conf/dlogenv.sh @@ -44,7 +44,7 @@ STREAM_NAME_PREFIX=distributedlog-smoketest # Benchmark Run Duration in minutes BENCHMARK_DURATION=60 # DL Namespace -DL_NAMESPACE=distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace +DL_NAMESPACE=distributedlog://127.0.0.1:2181/messaging/my_namespace # Benchmark SHARD id BENCHMARK_SHARD_ID=0 http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-core/bin/dlog ---------------------------------------------------------------------- diff --git a/distributedlog-core/bin/dlog b/distributedlog-core/bin/dlog index c4f9ed2..522e47c 100755 --- a/distributedlog-core/bin/dlog +++ b/distributedlog-core/bin/dlog @@ -20,8 +20,10 @@ set -e -DLOG_ROOT=$(git rev-parse --show-toplevel) -DLOG_HOME="${DLOG_ROOT}" +BASEDIR=$(dirname "$0") +DLOG_ROOT="${BASEDIR}/../.." + +DLOG_HOME="${DLOG_ROOT}/distributedlog-core" usage() { cat <<EOF @@ -46,8 +48,7 @@ These variable can also be set in conf/dlogenv.sh EOF } -cd "${DLOG_ROOT}" -source ./scripts/common.sh +source "${DLOG_ROOT}"/scripts/common.sh # get arguments COMMAND=$1 http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-service/bin/bundle ---------------------------------------------------------------------- diff --git a/distributedlog-service/bin/bundle b/distributedlog-service/bin/bundle index bd5b1a0..1b1904e 100755 --- a/distributedlog-service/bin/bundle +++ b/distributedlog-service/bin/bundle @@ -20,6 +20,9 @@ set -e -DLOG_ROOT=$(git rev-parse --show-toplevel) +BASEDIR=$(dirname "$0") +DLOG_ROOT="${BASEDIR}/../.." -bash "${DLOG_ROOT}"/scripts/bundle service +cd "${DLOG_ROOT}" + +bash scripts/bundle service http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-service/bin/dlog ---------------------------------------------------------------------- diff --git a/distributedlog-service/bin/dlog b/distributedlog-service/bin/dlog index b9a01e1..5a11611 100755 --- a/distributedlog-service/bin/dlog +++ b/distributedlog-service/bin/dlog @@ -20,8 +20,8 @@ set -e -DLOG_ROOT=$(git rev-parse --show-toplevel) - +BASEDIR=$(dirname "$0") +DLOG_ROOT="${BASEDIR}/.." DLOG_HOME="${DLOG_ROOT}" usage() { @@ -49,16 +49,12 @@ These variable can also be set in conf/dlogenv.sh EOF } -cd "${DLOG_ROOT}" -source ./scripts/common.sh +source "${DLOG_ROOT}"/../scripts/common.sh # get arguments COMMAND=$1 shift -#Change to DLOG_HOME to support relative paths -cd "$DLOG_HOME" - case "${COMMAND}" in local) exec java $OPTS $JMX_ARGS com.twitter.distributedlog.LocalDLMEmulator $@ http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/distributedlog-service/bin/dlog-daemon.sh ---------------------------------------------------------------------- diff --git a/distributedlog-service/bin/dlog-daemon.sh b/distributedlog-service/bin/dlog-daemon.sh index e683da0..976d4f1 100755 --- a/distributedlog-service/bin/dlog-daemon.sh +++ b/distributedlog-service/bin/dlog-daemon.sh @@ -19,26 +19,25 @@ # */ usage() { - cat <<EOF + cat <<EOF Usage: dlog-daemon.sh (start|stop) <service> <args...> where service is one of: - zookeeper Run the zookeeper server - bookie Run the bookie server - bookie-rereplicator Run the bookie rereplicator - writeproxy Run the write proxy server - writeproxy-monitor Run the write proxy monitor + zookeeper Run the zookeeper server + bookie Run the bookie server + bookie-rereplicator Run the bookie rereplicator + writeproxy Run the write proxy server + writeproxy-monitor Run the write proxy monitor where argument is one of: - -force (accepted only with stop service): Decides whether to stop the process forcefully if not stopped by normal shutdown + -force (accepted only with stop service): Decides whether to stop the process forcefully if not stopped by normal shutdown EOF } -BINDIR=`dirname "$0"` -DL_HOME=`cd $BINDIR/..;pwd` +BINDIR=$(dirname "$0") +DL_HOME=$(cd $BINDIR/..;pwd) -if [ -f $DL_HOME/conf/dlogenv.sh ] -then - . $DL_HOME/conf/dlogenv.sh +if [ -f "${DL_HOME}/conf/dlogenv.sh" ]; then + . "${DL_HOME}/conf/dlogenv.sh" fi SERVICE_PORT=${SERVICE_PORT:-"0"} @@ -47,56 +46,57 @@ SERVICE_ARGS="" # DLOG logging configuration DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DL_HOME/logs"} DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-'INFO,R'} + # Process Control Parameters DLOG_STOP_TIMEOUT=${DLOG_STOP_TIMEOUT:-30} DLOG_PID_DIR=${DLOG_PID_DIR:-$DL_HOME/pids} -if [ $# -lt 2 ] -then - echo "Error: no enough arguments provided." - usage - exit 1 +if [ $# -lt 2 ]; then + echo "Error: not enough arguments provided." + usage + exit 1 fi command=$1 shift + service=$1 shift service_class=$service case $service in - (zookeeper) - service_class="org.apache.zookeeper.server.quorum.QuorumPeerMain" - DLOG_ROOT_LOGGER=${ZK_ROOT_LOGGER:-'INFO,R'} - ;; - (bookie) - service_class="org.apache.bookkeeper.proto.BookieServer" - DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'} - ;; - (bookie-rereplicator) - service_class="org.apache.bookkeeper.replication.AutoRecoveryMain" - DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'} - ;; - (writeproxy) - service_class="com.twitter.distributedlog.service.DistributedLogServerApp" - DLOG_ROOT_LOGGER=${WP_ROOT_LOGGER:-'INFO,R'} - WP_CONF_FILE=${WP_CONF_FILE:-"$DL_HOME/conf/write_proxy.conf"} - WP_SERVICE_PORT=${WP_SERVICE_PORT:-'4181'} - WP_STATS_PORT=${WP_STATS_PORT:-'9000'} - WP_STATS_PROVIDER=${WP_STATS_PROVIDER:-'org.apache.bookkeeper.stats.CodahaleMetricsServletProvider'} - WP_SHARD_ID=${WP_SHARD_ID:-'0'} - WP_NAMESPACE=${WP_NAMESPACE:-'distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace'} - SERVICE_PORT=${WP_SERVICE_PORT} - SERVICE_ARGS="--conf ${WP_CONF_FILE} --uri ${WP_NAMESPACE} --shard-id ${WP_SHARD_ID} --port ${WP_SERVICE_PORT} --stats-port ${WP_STATS_PORT} --stats-provider ${WP_STATS_PROVIDER} --announce --thriftmux" - DLOG_EXTRA_OPTS="${DLOG_EXTRA_OPTS} -DcodahaleStatsHttpPort=${WP_STATS_PORT} -Dserver_port=${WP_SERVICE_PORT} -Dserver_shard=${WP_SHARD_ID}" - ;; - (writeproxy-monitor) - ;; - (*) - echo "Error: unknown service name $service" - usage - exit 1 - ;; + (zookeeper) + service_class="org.apache.zookeeper.server.quorum.QuorumPeerMain" + DLOG_ROOT_LOGGER=${ZK_ROOT_LOGGER:-'INFO,R'} + ;; + (bookie) + service_class="org.apache.bookkeeper.proto.BookieServer" + DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'} + ;; + (bookie-rereplicator) + service_class="org.apache.bookkeeper.replication.AutoRecoveryMain" + DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'} + ;; + (writeproxy) + service_class="com.twitter.distributedlog.service.DistributedLogServerApp" + DLOG_ROOT_LOGGER=${WP_ROOT_LOGGER:-'INFO,R'} + WP_CONF_FILE=${WP_CONF_FILE:-"$DL_HOME/conf/write_proxy.conf"} + WP_SERVICE_PORT=${WP_SERVICE_PORT:-'4181'} + WP_STATS_PORT=${WP_STATS_PORT:-'9000'} + WP_STATS_PROVIDER=${WP_STATS_PROVIDER:-'org.apache.bookkeeper.stats.CodahaleMetricsServletProvider'} + WP_SHARD_ID=${WP_SHARD_ID:-'0'} + WP_NAMESPACE=${WP_NAMESPACE:-'distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace'} + SERVICE_PORT=${WP_SERVICE_PORT} + SERVICE_ARGS="--conf ${WP_CONF_FILE} --uri ${WP_NAMESPACE} --shard-id ${WP_SHARD_ID} --port ${WP_SERVICE_PORT} --stats-port ${WP_STATS_PORT} --stats-provider ${WP_STATS_PROVIDER} --announce --thriftmux" + DLOG_EXTRA_OPTS="${DLOG_EXTRA_OPTS} -DcodahaleStatsHttpPort=${WP_STATS_PORT} -Dserver_port=${WP_SERVICE_PORT} -Dserver_shard=${WP_SHARD_ID}" + ;; + (writeproxy-monitor) + ;; + (*) + echo "Error: unknown service name $service" + usage + exit 1 + ;; esac echo "doing $command $service ..." @@ -110,21 +110,20 @@ pid=$DLOG_PID_DIR/dlog-$service-$SERVICE_PORT.pid out=$DLOG_LOG_DIR/dlog-$service-$HOSTNAME-$SERVICE_PORT.out logfile=$DLOG_LOG_DIR/$DLOG_LOG_FILE -rotate_out_log () -{ - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" - num=$prev - done - mv "$log" "$log.$num"; - fi +rotate_out_log () { + log=$1; + num=5; + if [ -n "$2" ]; then + num=$2 + fi + if [ -f "$log" ]; then # rotate logs + while [ $num -gt 1 ]; do + prev=`expr $num - 1` + [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" + num=$prev + done + mv "$log" "$log.$num"; + fi } mkdir -p "$DLOG_LOG_DIR" @@ -133,8 +132,8 @@ mkdir -p "$DLOG_PID_DIR" case $command in (start) if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $service running as process `cat $pid`. Stop it first. + if kill -0 $(cat $pid) > /dev/null 2>&1; then + echo $service running as process $(cat $pid). Stop it first. exit 1 fi fi @@ -153,41 +152,36 @@ case $command in (stop) if [ -f $pid ]; then - TARGET_PID=`cat $pid` + TARGET_PID=$(cat $pid) if kill -0 $TARGET_PID > /dev/null 2>&1; then echo stopping $service kill $TARGET_PID count=0 location=$DLOG_LOG_DIR - while ps -p $TARGET_PID > /dev/null; - do + while ps -p $TARGET_PID > /dev/null; do echo "Shutdown is in progress... Please wait..." sleep 1 - count=`expr $count + 1` - + count=$(expr $count + 1) if [ "$count" = "$DLOG_STOP_TIMEOUT" ]; then - break + break fi - done - + done if [ "$count" != "$DLOG_STOP_TIMEOUT" ]; then - echo "Shutdown completed." + echo "Shutdown completed." fi - if kill -0 $TARGET_PID > /dev/null 2>&1; then - fileName=$location/$service.out - $JAVA_HOME/bin/jstack $TARGET_PID > $fileName - echo Thread dumps are taken for analysis at $fileName - if [ "$1" == "-force" ] - then - echo forcefully stopping $service - kill -9 $TARGET_PID >/dev/null 2>&1 - echo Successfully stopped the process - else - echo "WARNNING : $service is not stopped completely." - exit 1 - fi + fileName=$location/$service.out + $JAVA_HOME/bin/jstack $TARGET_PID > $fileName + echo Thread dumps are taken for analysis at $fileName + if [ "$1" == "-force" ]; then + echo forcefully stopping $service + kill -9 $TARGET_PID >/dev/null 2>&1 + echo Successfully stopped the process + else + echo "WARNNING : $service is not stopped completely." + exit 1 + fi fi else echo no $service to stop http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index f517914..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -FROM java:8 - -MAINTAINER Arvind Kandhare ([email protected]) - -COPY .. /opt/distributedlog-trunk/ - -ENV BROKER_ID 0 -ENV ZK_SERVERS zk1:2181 - -EXPOSE 3181 -EXPOSE 9001 -EXPOSE 4181 -EXPOSE 20001 - -CMD ["/opt/distributedlog-trunk/dist/release/vagrant/bk.sh", $BROKER_ID, "127.0.0.1", $ZK_SERVERS ] - - - http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/docs/admin_guide/vagrant.rst ---------------------------------------------------------------------- diff --git a/docs/admin_guide/vagrant.rst b/docs/admin_guide/vagrant.rst index 70e6fb0..e4ed574 100644 --- a/docs/admin_guide/vagrant.rst +++ b/docs/admin_guide/vagrant.rst @@ -13,6 +13,6 @@ Steps 1. Create a snapshot using ./scripts/snapshot 2. Run vagrant up from the root directory of the enlistment 3. Vagrant brings up a zookeepers with machine names: zk1,zk2, .... with IP addresses 192.168.50.11,192.168.50.12,.... -4. Vagrant brings the bookies with machine names: node1,node2, ... with IP addresses 192.168.50.51,192.168.50.52,.... -5. The script will also start writeproxies at distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace as the namespace. If you want it to point to a different namespace/port/shard, please update the ./vagrant/bk.sh script. +4. Vagrant brings the bookies with machine names: bk1,bk2, ... with IP addresses 192.168.50.51,192.168.50.52,.... +5. The script will also start writeproxies at distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace as the namespace. If you want it to point to a different namespace/port/shard, please update the ./vagrant/bk.sh script. 6. If you want to run the client on the host machine, please add these node names and their IP addresses to the /etc/hosts file on the host. http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/docs/deployment/cluster.rst ---------------------------------------------------------------------- diff --git a/docs/deployment/cluster.rst b/docs/deployment/cluster.rst index 8dcd461..250ba3c 100644 --- a/docs/deployment/cluster.rst +++ b/docs/deployment/cluster.rst @@ -38,8 +38,9 @@ Or run `./scripts/snapshot` to build the release packages from current source. T packages contain the binaries for running `distributedlog-service`, `distributedlog-benchmark` and `distributedlog-tutorials`. -NOTE: we run following instructions from distributedlog source code after running `mvn clean install`. -And assume `DL_HOME` is the directory of distributedlog source. +NOTE: we run the following instructions from distributedlog source code after +running `mvn clean install`. And assume `DL_HOME` is the directory of +distributedlog source. Zookeeper --------- @@ -118,9 +119,10 @@ First of all, choose the zookeeper cluster that the bookies will use and set `zk the configuration files. :: - + zkServers=localhost:2181 + Choose the zookeeper path to store bookkeeper metadata and set `zkLedgersRootPath` in the configuration files. Let's use `/messaging/bookkeeper/ledgers` in this instruction. @@ -159,7 +161,7 @@ So using `zkshell` to create the `zkLedgersRootPath`. If the `zkLedgersRootPath`, run `metaformat` to format the bookkeeper metadata. :: - + $ BOOKIE_CONF=${DL_HOME}/distributedlog-service/conf/bookie-1.conf ./distributedlog-service/bin/dlog bkshell metaformat Are you sure to format bookkeeper metadata ? (Y or N) Y @@ -176,7 +178,7 @@ Configure the ports that used by bookies. bookie-1: :: - + # Port that bookie server listen on bookiePort=3181 # Exporting codahale stats @@ -185,7 +187,7 @@ bookie-1: bookie-2: :: - + # Port that bookie server listen on bookiePort=3182 # Exporting codahale stats @@ -194,7 +196,7 @@ bookie-2: bookie-3: :: - + # Port that bookie server listen on bookiePort=3183 # Exporting codahale stats @@ -206,7 +208,7 @@ Configure Disk Layout Configure the disk directories used by a bookie server by setting following options. :: - + # Directory Bookkeeper outputs its write ahead log journalDirectory=/tmp/data/bk/journal # Directory Bookkeeper outputs ledger snapshots @@ -219,7 +221,7 @@ As we are configuring a 3-nodes bookkeeper cluster, we modify the following sett bookie-1: :: - + # Directory Bookkeeper outputs its write ahead log journalDirectory=/tmp/data/bk-1/journal # Directory Bookkeeper outputs ledger snapshots @@ -230,7 +232,7 @@ bookie-1: bookie-2: :: - + # Directory Bookkeeper outputs its write ahead log journalDirectory=/tmp/data/bk-2/journal # Directory Bookkeeper outputs ledger snapshots @@ -241,7 +243,7 @@ bookie-2: bookie-3: :: - + # Directory Bookkeeper outputs its write ahead log journalDirectory=/tmp/data/bk-3/journal # Directory Bookkeeper outputs ledger snapshots @@ -256,7 +258,7 @@ Once the disk directories are configured correctly in the configuration file, us `bkshell bookieformat` to format the bookie. :: - + BOOKIE_CONF=${DL_HOME}/distributedlog-service/conf/bookie-1.conf ./distributedlog-service/bin/dlog bkshell bookieformat BOOKIE_CONF=${DL_HOME}/distributedlog-service/conf/bookie-2.conf ./distributedlog-service/bin/dlog bkshell bookieformat BOOKIE_CONF=${DL_HOME}/distributedlog-service/conf/bookie-3.conf ./distributedlog-service/bin/dlog bkshell bookieformat @@ -268,16 +270,16 @@ Start bookie Start the bookie using `dlog-daemon.sh`. :: - + SERVICE_PORT=3181 ./distributedlog-service/bin/dlog-daemon.sh start bookie --conf ${DL_HOME}/distributedlog-service/conf/bookie-1.conf SERVICE_PORT=3182 ./distributedlog-service/bin/dlog-daemon.sh start bookie --conf ${DL_HOME}/distributedlog-service/conf/bookie-2.conf SERVICE_PORT=3183 ./distributedlog-service/bin/dlog-daemon.sh start bookie --conf ${DL_HOME}/distributedlog-service/conf/bookie-3.conf - + Verify whether the bookie is setup correctly. You could simply check whether the bookie is showed up in zookeeper `zkLedgersRootPath`/available znode. :: - + $ ./distributedlog-service/bin/dlog zkshell localhost:2181 Connecting to localhost:2181 Welcome to ZooKeeper! @@ -294,7 +296,7 @@ zookeeper `zkLedgersRootPath`/available znode. Or check if the bookie is exposing the stats at port `codahaleStatsHttpPort`. :: - + // ping the service $ curl localhost:9001/ping pong @@ -307,7 +309,7 @@ Stop bookie Stop the bookie using `dlog-daemon.sh`. :: - + $ ./distributedlog-service/bin/dlog-daemon.sh stop bookie // Example: $ SERVICE_PORT=3181 ./distributedlog-service/bin/dlog-daemon.sh stop bookie @@ -322,13 +324,13 @@ Turn bookie to readonly Start the bookie in `readonly` mode. :: - + $ SERVICE_PORT=3181 ./distributedlog-service/bin/dlog-daemon.sh start bookie --conf ${DL_HOME}/distributedlog-service/conf/bookie-1.conf --readonly Verify if the bookie is running in `readonly` mode. :: - + $ ./distributedlog-service/bin/dlog zkshell localhost:2181 Connecting to localhost:2181 Welcome to ZooKeeper! @@ -376,7 +378,7 @@ Create namespace binding using `dlog tool`. For example, we create a namespace bookkeeper cluster we just created above. :: - + $ distributedlog-service/bin/dlog admin bind \\ -dlzr 127.0.0.1:2181 \\ -dlzw 127.0.0.1:2181 \\ @@ -425,7 +427,7 @@ Run write proxy A write proxy could be started using `dlog-daemon.sh` script under `distributedlog-service`. :: - + WP_SHARD_ID=${WP_SHARD_ID} WP_SERVICE_PORT=${WP_SERVICE_PORT} WP_STATS_PORT=${WP_STATS_PORT} ./distributedlog-service/bin/dlog-daemon.sh start writeproxy - `WP_SHARD_ID`: A non-negative integer. You don't need to guarantee uniqueness of shard id, as it is just an @@ -442,7 +444,7 @@ Please check `distributedlog-service/conf/dlogenv.sh` for more environment varia For example, we start 3 write proxies locally and point to the namespace created above. :: - + $ WP_SHARD_ID=1 WP_SERVICE_PORT=4181 WP_STATS_PORT=20001 ./distributedlog-service/bin/dlog-daemon.sh start writeproxy $ WP_SHARD_ID=2 WP_SERVICE_PORT=4182 WP_STATS_PORT=20002 ./distributedlog-service/bin/dlog-daemon.sh start writeproxy $ WP_SHARD_ID=3 WP_SERVICE_PORT=4183 WP_STATS_PORT=20003 ./distributedlog-service/bin/dlog-daemon.sh start writeproxy @@ -452,7 +454,7 @@ The write proxy will announce itself to the zookeeper path `.write_proxy` under We could verify that the write proxy is running correctly by checking the zookeeper path or checking its stats port. :: - + $ ./distributedlog-service/bin/dlog zkshell localhost:2181 Connecting to localhost:2181 Welcome to ZooKeeper! @@ -466,7 +468,7 @@ We could verify that the write proxy is running correctly by checking the zookee :: - + $ curl localhost:20001/ping pong @@ -477,7 +479,7 @@ Add and Remove Write Proxies Removing a write proxy is pretty straightforward by just killing the process. :: - + WP_SHARD_ID=1 WP_SERVICE_PORT=4181 WP_STATS_PORT=10001 ./distributedlog-service/bin/dlog-daemon.sh stop writeproxy @@ -506,14 +508,14 @@ Create 10 streams. Tail read from the 10 streams. :: - + $ ./distributedlog-tutorials/distributedlog-basic/bin/runner run com.twitter.distributedlog.basic.MultiReader distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace stream-0,stream-1,stream-2,stream-3,stream-4,stream-5,stream-6,stream-7,stream-8,stream-9,stream-10 Run record generator over some streams :: - + $ ./distributedlog-tutorials/distributedlog-basic/bin/runner run com.twitter.distributedlog.basic.RecordGenerator 'zk!127.0.0.1:2181!/messaging/distributedlog/mynamespace/.write_proxy' stream-0 100 $ ./distributedlog-tutorials/distributedlog-basic/bin/runner run com.twitter.distributedlog.basic.RecordGenerator 'zk!127.0.0.1:2181!/messaging/distributedlog/mynamespace/.write_proxy' stream-1 100 @@ -521,7 +523,7 @@ Run record generator over some streams Check the terminal running `MultiReader`. You will see similar output as below: :: - + """ Received record DLSN{logSegmentSequenceNo=1, entryId=21044, slotId=0} from stream stream-0 """ http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/docs/deployment/docker.rst ---------------------------------------------------------------------- diff --git a/docs/deployment/docker.rst b/docs/deployment/docker.rst index 8222dd6..d9fd87a 100644 --- a/docs/deployment/docker.rst +++ b/docs/deployment/docker.rst @@ -24,7 +24,7 @@ Steps ./scripts/snapshot -2. Create your own docker image using +2. Create your own docker image using .. code-block:: bash http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/scripts/bundle ---------------------------------------------------------------------- diff --git a/scripts/bundle b/scripts/bundle index a0c8a3a..54691ec 100755 --- a/scripts/bundle +++ b/scripts/bundle @@ -27,8 +27,6 @@ if [[ -z "${COMPONENT}" ]]; then exit 1 fi -cd "${DLOG_ROOT}" - rm -rf "distributedlog-${COMPONENT}/lib" rm -rf "dist/distributedlog-${COMPONENT}*" http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/scripts/common.sh ---------------------------------------------------------------------- diff --git a/scripts/common.sh b/scripts/common.sh index 433f93b..f37ed1b 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -48,6 +48,7 @@ else echo "JMX disabled by user request" >&2 fi +echo "DLOG_HOME => ${DLOG_HOME}" DEFAULT_LOG_CONF="${DLOG_HOME}/conf/log4j.properties" [ -f "${DLOG_HOME}/conf/dlogenv.sh" ] && source "${DLOG_HOME}/conf/dlogenv.sh" @@ -59,7 +60,7 @@ if [ $? == 0 ]; then fi # exclude tests jar -BUILT_JAR=$(ls "${DLOG_HOME}/target/distributedlog-*.jar" 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1) +BUILT_JAR=$(ls "${DLOG_HOME}"/target/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1) if [ -e "${BUILD_JAR}" ] && [ -e "${DLOG_JAR}" ]; then echo "\nCouldn't find dlog jar."; @@ -93,9 +94,8 @@ else add_maven_deps_to_classpath fi -# if no args specified, show usage +# if no args specified, exit if [ $# = 0 ]; then - usage exit 1 fi http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/scripts/runner ---------------------------------------------------------------------- diff --git a/scripts/runner b/scripts/runner index cc05946..937829a 100755 --- a/scripts/runner +++ b/scripts/runner @@ -22,7 +22,8 @@ set -e -DLOG_ROOT=$(git rev-parse --show-toplevel) +BASEDIR=$(dirname "$0") +DLOG_ROOT="${BASEDIR}/.." DLOG_HOME="${DLOG_ROOT}/distributedlog-service" http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/scripts/snapshot ---------------------------------------------------------------------- diff --git a/scripts/snapshot b/scripts/snapshot index 2c9e285..2f2f732 100755 --- a/scripts/snapshot +++ b/scripts/snapshot @@ -32,7 +32,7 @@ rm -rf dist/* # create the layout for module in distributedlog-service distributedlog-benchmark distributedlog-tutorials/distributedlog-basic distributedlog-tutorials/distributedlog-messaging; do - mkdir -p "dist/release/${module}/{bin,conf,lib}" + mkdir -p "dist/release/${module}"/{bin,conf,lib} done # build the package @@ -42,26 +42,27 @@ mvn clean install -Ptwitter-ostrich-provider -DskipTests cp -rf distributedlog-service/bin/* dist/release/distributedlog-service/bin cp -rf distributedlog-service/conf/* dist/release/distributedlog-service/conf cp -rf distributedlog-service/lib/* dist/release/distributedlog-service/lib -cp -rf distributedlog-service/target/distributedlog-service-*.jar dist/release/distributedlog-service +cp -rf distributedlog-service/target/distributedlog-service-*.jar dist/release/distributedlog-service/lib cp -rf vagrant dist/release/ +cp -rf scripts dist/release/ # build the distributedlog-benchmark cp -rf distributedlog-benchmark/bin/* dist/release/distributedlog-benchmark/bin cp -rf distributedlog-benchmark/conf/* dist/release/distributedlog-benchmark/conf cp -rf distributedlog-benchmark/lib/* dist/release/distributedlog-benchmark/lib -cp -rf distributedlog-benchmark/target/distributedlog-benchmark-*.jar dist/release/distributedlog-benchmark +cp -rf distributedlog-benchmark/target/distributedlog-benchmark-*.jar dist/release/distributedlog-benchmark/lib # build the distributedlog-tutorial/basic cp -rf distributedlog-tutorials/distributedlog-basic/bin/* dist/release/distributedlog-tutorials/distributedlog-basic/bin cp -rf distributedlog-tutorials/distributedlog-basic/conf/* dist/release/distributedlog-tutorials/distributedlog-basic/conf cp -rf distributedlog-tutorials/distributedlog-basic/lib/* dist/release/distributedlog-tutorials/distributedlog-basic/lib -cp -rf distributedlog-tutorials/distributedlog-basic/target/distributedlog-basic-*.jar dist/release/distributedlog-tutorials/distributedlog-basic +cp -rf distributedlog-tutorials/distributedlog-basic/target/distributedlog-basic-*.jar dist/release/distributedlog-tutorials/distributedlog-basic/lib # build the distributedlog-tutorial/messaging cp -rf distributedlog-tutorials/distributedlog-messaging/bin/* dist/release/distributedlog-tutorials/distributedlog-messaging/bin cp -rf distributedlog-tutorials/distributedlog-messaging/conf/* dist/release/distributedlog-tutorials/distributedlog-messaging/conf cp -rf distributedlog-tutorials/distributedlog-messaging/lib/* dist/release/distributedlog-tutorials/distributedlog-messaging/lib -cp -rf distributedlog-tutorials/distributedlog-messaging/target/distributedlog-messaging-*.jar dist/release/distributedlog-tutorials/distributedlog-messaging +cp -rf distributedlog-tutorials/distributedlog-messaging/target/distributedlog-messaging-*.jar dist/release/distributedlog-tutorials/distributedlog-messaging/lib # release cd dist/release http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/vagrant/base.sh ---------------------------------------------------------------------- diff --git a/vagrant/base.sh b/vagrant/base.sh index 03e348e..21e6167 100755 --- a/vagrant/base.sh +++ b/vagrant/base.sh @@ -18,39 +18,40 @@ set -e if [ -z $(which javac) ]; then - apt-get -y update - apt-get install -y --no-install-recommends software-properties-common python-software-properties - add-apt-repository -y ppa:webupd8team/java - apt-get -y update - - # Try to share cache. See Vagrantfile for details - mkdir -p /var/cache/oracle-jdk8-installer - if [ -e "/tmp/oracle-jdk8-installer-cache/" ]; then - find /tmp/oracle-jdk8-installer-cache/ -not -empty -exec cp '{}' /var/cache/oracle-jdk8-installer/ \; - fi - - /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections - apt-get -y install --force-yes --no-install-recommends oracle-java8-installer oracle-java8-set-default - - if [ -e "/tmp/oracle-jdk8-installer-cache/" ]; then - cp -R /var/cache/oracle-jdk8-installer/* /tmp/oracle-jdk8-installer-cache - fi + apt-get -y update + apt-get install -y --no-install-recommends software-properties-common python-software-properties + add-apt-repository -y ppa:webupd8team/java + apt-get -y update + + # Try to share cache. See Vagrantfile for details + mkdir -p /var/cache/oracle-jdk8-installer + if [ -e "/tmp/oracle-jdk8-installer-cache/" ]; then + find /tmp/oracle-jdk8-installer-cache/ -not -empty -exec cp '{}' /var/cache/oracle-jdk8-installer/ \; + fi + + /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections + apt-get -y install --force-yes --no-install-recommends oracle-java8-installer oracle-java8-set-default maven2 + + if [ -e "/tmp/oracle-jdk8-installer-cache/" ]; then + cp -R /var/cache/oracle-jdk8-installer/* /tmp/oracle-jdk8-installer-cache + fi fi chmod a+rw /opt if [ -h /opt/distributedlog-trunk ]; then - # reset symlink - rm /opt/distributedlog-trunk + # reset symlink + rm /opt/distributedlog-trunk fi -ln -s /vagrant /opt/distributedlog-trunk +ln -s /vagrant /opt/distributedlog-trunk # For EC2 nodes, we want to use /mnt, which should have the local disk. On local # VMs, we can just create it if it doesn't exist and use it like we'd use # /tmp. Eventually, we'd like to also support more directories, e.g. when EC2 # instances have multiple local disks. if [ ! -e /mnt ]; then - mkdir /mnt + mkdir /mnt fi + chmod a+rwx /mnt http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/vagrant/bk.sh ---------------------------------------------------------------------- diff --git a/vagrant/bk.sh b/vagrant/bk.sh index 3c0009a..1eb5ab1 100755 --- a/vagrant/bk.sh +++ b/vagrant/bk.sh @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - # Usage: bk.sh <broker ID> <public hostname or IP> <list zookeeper public hostname or IP + port> set -e @@ -40,47 +39,54 @@ if [ ! -z $DEPLOY_BOTH ]; then DEPLOY_WP=true fi - log_dir=/opt/distributedlog-trunk/dist/release cd $log_dir #Stop the running bookie -echo "Killing server" +echo "Killing bookkeeper server" ./distributedlog-service/bin/dlog-daemon.sh stop bookie cp ./distributedlog-service/conf/bookie.conf.template ./distributedlog-service/conf/bookie-$BROKER_ID.conf sed \ - -e 's/zkServers=localhost:2181/'zkServers=$PUBLIC_ZOOKEEPER_ADDRESSES'/' \ - ./distributedlog-service/conf/bookie.conf.template > ./distributedlog-service/conf/bookie-$BROKER_ID.conf + -e 's/zkServers=localhost:2181/'zkServers=$PUBLIC_ZOOKEEPER_ADDRESSES'/' \ + ./distributedlog-service/conf/bookie.conf.template > ./distributedlog-service/conf/bookie-$BROKER_ID.conf -sleep 5 +sleep 5 if [ $DEPLOY_BK == true ]; then - if [ $BROKER_ID -eq "1" ]; then - echo "create /messaging" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES - echo "create /messaging/bookkeeper" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES - echo "create /messaging/bookkeeper/ledgers" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES - echo "Metafirmatting bookie" - export BOOKIE_CONF=$log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf - echo "Y" | ./distributedlog-service/bin/dlog bkshell metaformat - fi - - echo "Configuring bookkeeper" - BOOKIE_CONF=$log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf ./distributedlog-service/bin/dlog bkshell bookieformat - - echo "Starting server" - SERVICE_PORT=3181 ./distributedlog-service/bin/dlog-daemon.sh start bookie -c $log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf - - - if [ $BROKER_ID -eq "1" ]; then - ./distributedlog-service/bin/dlog admin bind -dlzr $PUBLIC_ZOOKEEPER_ADDRESSES -dlzw $PUBLIC_ZOOKEEPER_ADDRESSES -s $PUBLIC_ZOOKEEPER_ADDRESSES -bkzr $PUBLIC_ZOOKEEPER_ADDRESSES \ - -l /messaging/bookkeeper/ledgers -i false -r true -c distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace - fi - + if [ $BROKER_ID -eq "1" ]; then + echo "create /messaging" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES + echo "create /messaging/bookkeeper" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES + echo "create /messaging/bookkeeper/ledgers" | ./distributedlog-service/bin/dlog zkshell $PUBLIC_ZOOKEEPER_ADDRESSES + echo "Metaformatting bookie" + export BOOKIE_CONF=$log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf + ./distributedlog-service/bin/dlog bkshell metaformat -n -f + fi + + echo "Configuring bookkeeper" + BOOKIE_CONF=$log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf ./distributedlog-service/bin/dlog bkshell bookieformat + + echo "Starting server" + SERVICE_PORT=3181 ./distributedlog-service/bin/dlog-daemon.sh start bookie -c $log_dir/distributedlog-service/conf/bookie-$BROKER_ID.conf + + if [ $BROKER_ID -eq "1" ]; then + ./distributedlog-service/bin/dlog admin bind \ + -dlzr $PUBLIC_ZOOKEEPER_ADDRESSES \ + -dlzw $PUBLIC_ZOOKEEPER_ADDRESSES \ + -s $PUBLIC_ZOOKEEPER_ADDRESSES \ + -bkzr $PUBLIC_ZOOKEEPER_ADDRESSES \ + -l /messaging/bookkeeper/ledgers \ + -i false \ + -r true \ + -c distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace + fi fi - if [ $DEPLOY_WP == true ]; then - WP_NAMESPACE=distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace WP_SHARD_ID=$BROKER_ID WP_SERVICE_PORT=4181 WP_STATS_PORT=20001 ./distributedlog-service/bin/dlog-daemon.sh start writeproxy + WP_NAMESPACE=distributedlog://$PUBLIC_ZOOKEEPER_ADDRESSES/messaging/distributedlog/mynamespace \ + WP_SHARD_ID=$BROKER_ID \ + WP_SERVICE_PORT=4181 \ + WP_STATS_PORT=20001 \ + ./distributedlog-service/bin/dlog-daemon.sh start writeproxy fi http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/927f0ca0/vagrant/zk.sh ---------------------------------------------------------------------- diff --git a/vagrant/zk.sh b/vagrant/zk.sh index 3d5d5de..c690149 100755 --- a/vagrant/zk.sh +++ b/vagrant/zk.sh @@ -30,9 +30,9 @@ echo "initLimit=5" >> $log_dir/distributedlog-service/conf/zookeeper-$ZKID.prope echo "syncLimit=2" >> $log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties echo "quorumListenOnAllIPs=true" >> $log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties sed '/server.1/d' \ - ./distributedlog-service/conf/bookie.conf.template > ./distributedlog-service/conf/zookeeper-$ZKID.conf + ./distributedlog-service/conf/bookie.conf.template > ./distributedlog-service/conf/zookeeper-$ZKID.conf for i in $(seq 1 $NUM_ZK); do - echo "server.${i}=zk${i}:2888:3888" >>$log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties + echo "server.${i}=zk${i}:2888:3888" >>$log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties done mkdir -p /tmp/data/zookeeper @@ -41,6 +41,7 @@ echo "$ZKID" > /tmp/data/zookeeper/myid echo "Killing ZooKeeper" ./distributedlog-service/bin/dlog-daemon.sh stop zookeeper || true sleep 5 + echo "Starting ZooKeeper" echo "./distributedlog-service/bin/dlog-daemon.sh start zookeeper $log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties" ./distributedlog-service/bin/dlog-daemon.sh start zookeeper $log_dir/distributedlog-service/conf/zookeeper-$ZKID.properties
