Repository: incubator-distributedlog Updated Branches: refs/heads/master 927f0ca01 -> dc1bb30ba
DL-141: Fix broken scripts in tutorials and add integration test to travis ci - fix the broken scripts used in tutorials - fix the write_proxy configuration (a change was merged to validate configuration which it brokes the configuration too) - add integration test to validate the basic end-to-end functionalities Author: Sijie Guo <[email protected]> Reviewers: Leigh Stewart <[email protected]> Closes #76 from sijie/sijie/script_for_ci Project: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/commit/dc1bb30b Tree: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/tree/dc1bb30b Diff: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/diff/dc1bb30b Branch: refs/heads/master Commit: dc1bb30ba3ad2ab0898cf7f074261f1de0bd2d79 Parents: 927f0ca Author: Sijie Guo <[email protected]> Authored: Fri Dec 16 11:15:21 2016 -0800 Committer: Sijie Guo <[email protected]> Committed: Fri Dec 16 11:15:21 2016 -0800 ---------------------------------------------------------------------- .travis.yml | 4 + distributedlog-core/bin/dlog | 3 - distributedlog-core/conf/write_proxy.conf | 2 +- .../distributedlog-kafka/bin/runner | 1 - scripts/common.sh | 6 +- scripts/integration/smoketest.sh | 113 +++++++++++++++++++ scripts/runner | 11 +- 7 files changed, 124 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 50343d3..a064f24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,10 @@ script: - travis_retry mvn --batch-mode clean apache-rat:check - travis_wait 60 mvn --batch-mode clean package findbugs:check +# run integration test after success +after_success: + - ./scripts/integration/smoketest.sh + cache: directories: - $HOME/.m2 http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/distributedlog-core/bin/dlog ---------------------------------------------------------------------- diff --git a/distributedlog-core/bin/dlog b/distributedlog-core/bin/dlog index 522e47c..6632fd0 100755 --- a/distributedlog-core/bin/dlog +++ b/distributedlog-core/bin/dlog @@ -54,9 +54,6 @@ source "${DLOG_ROOT}"/scripts/common.sh COMMAND=$1 shift -#Change to DLOG_HOME to support relative paths -cd "$DLOG_HOME" - case "${COMMAND}" in local) exec java $OPTS $JMX_ARGS com.twitter.distributedlog.LocalDLMEmulator $@ http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/distributedlog-core/conf/write_proxy.conf ---------------------------------------------------------------------- diff --git a/distributedlog-core/conf/write_proxy.conf b/distributedlog-core/conf/write_proxy.conf index adad68a..271f177 100644 --- a/distributedlog-core/conf/write_proxy.conf +++ b/distributedlog-core/conf/write_proxy.conf @@ -39,7 +39,7 @@ bkcZKRetryMaxBackoffMillis=200 # bookkeeper client timeouts bkcWriteTimeoutSeconds=2 -bkcReadTimeoutSeconds=1 +bkcReadTimeoutSeconds=2 bkcNumWorkerThreads=32 bkc.numChannelsPerBookie=1 bkc.enableTaskExecutionStats=true http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/distributedlog-tutorials/distributedlog-kafka/bin/runner ---------------------------------------------------------------------- diff --git a/distributedlog-tutorials/distributedlog-kafka/bin/runner b/distributedlog-tutorials/distributedlog-kafka/bin/runner deleted file mode 120000 index a0ce2ef..0000000 --- a/distributedlog-tutorials/distributedlog-kafka/bin/runner +++ /dev/null @@ -1 +0,0 @@ -../../../scripts/runner \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/scripts/common.sh ---------------------------------------------------------------------- diff --git a/scripts/common.sh b/scripts/common.sh index f37ed1b..8df5248 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -54,13 +54,13 @@ DEFAULT_LOG_CONF="${DLOG_HOME}/conf/log4j.properties" [ -f "${DLOG_HOME}/conf/dlogenv.sh" ] && source "${DLOG_HOME}/conf/dlogenv.sh" # exclude tests jar -RELEASE_JAR=$(ls "${DLOG_HOME}/distributedlog-*.jar" 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1) +RELEASE_JAR=$(ls "${DLOG_HOME}/distributedlog-*.jar" 2> /dev/null | grep -v 'tests|javadoc|sources' | tail -1) if [ $? == 0 ]; then DLOG_JAR="${RELEASE_JAR}" fi # exclude tests jar -BUILT_JAR=$(ls "${DLOG_HOME}"/target/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1) +BUILT_JAR=$(ls "${DLOG_HOME}"/target/distributedlog-*.jar 2> /dev/null | grep -v 'tests|javadoc|sources' | tail -1) if [ -e "${BUILD_JAR}" ] && [ -e "${DLOG_JAR}" ]; then echo "\nCouldn't find dlog jar."; @@ -79,7 +79,7 @@ add_maven_deps_to_classpath() { # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${DLOG_HOME}/target/cached_classpath.txt" + f="${PWD}/${DLOG_HOME}/target/cached_classpath.txt" if [ ! -f "${f}" ]; then "${MVN}" -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null fi http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/scripts/integration/smoketest.sh ---------------------------------------------------------------------- diff --git a/scripts/integration/smoketest.sh b/scripts/integration/smoketest.sh new file mode 100755 index 0000000..5141756 --- /dev/null +++ b/scripts/integration/smoketest.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +set -e + +BASEDIR=$(dirname "$0") +DLOG_ROOT="${BASEDIR}/../.." + +LOG_DIR="${DLOG_ROOT}/logs" +mkdir -p ${LOG_DIR} + +# DL ZK & Namespace +NAMESPACE="${USER}-${RANDOM}" +ZK_PORT=7000 + +# write proxies +export WP_SHARD_ID=1 +export WP_SERVICE_PORT=8000 +export WP_STATS_PORT=8001 +export WP_NAMESPACE="distributedlog://127.0.0.1:${ZK_PORT}/messaging/${NAMESPACE}" + +# streams +SMOKESTREAM_PREFIX="smoketest-stream-" + +# start the sandbox +nohup ${DLOG_ROOT}/distributedlog-service/bin/dlog local ${ZK_PORT} > ${LOG_DIR}/sandbox.out 2>&1& +echo $! > ${LOG_DIR}/sandbox.pid + +# create namespace +${DLOG_ROOT}/distributedlog-core/bin/dlog admin bind -l /ledgers -s 127.0.0.1:${ZK_PORT} -c distributedlog://127.0.0.1:${ZK_PORT}/messaging/${NAMESPACE} + +if [ $? -ne 0 ]; then + echo "Failed to create namespace '${NAMESPACE}'." + exit 1 +fi + +# create streams +${DLOG_ROOT}/distributedlog-service/bin/dlog tool create -u distributedlog://127.0.0.1:${ZK_PORT}/messaging/${NAMESPACE} -r ${SMOKESTREAM_PREFIX} -e 1-5 -f + +if [ $? -ne 0 ]; then + echo "Failed to create streams prefixed with '${SMOKESTREAM_PREFIX}'." + exit 1 +fi + +# list streams +STREAM_OUTPUT=$(${DLOG_ROOT}/distributedlog-service/bin/dlog tool list -u distributedlog://127.0.0.1:${ZK_PORT}/messaging/${NAMESPACE} | grep "${SMOKESTREAM_PREFIX}") + +echo "Create streams : ${STREAM_OUTPUT}." + +for i in {1..5}; do + if [[ ${STREAM_OUTPUT} != *"${SMOKESTREAM_PREFIX}${i}"* ]]; then + echo "Stream '${SMOKESTREAM_PREFIX}${i}' is not created." + exit 1 + fi +done + +# start a write proxy +${DLOG_ROOT}/distributedlog-service/bin/dlog-daemon.sh start writeproxy + +# tail the the streams +nohup ${DLOG_ROOT}/distributedlog-tutorials/distributedlog-basic/bin/runner run com.twitter.distributedlog.basic.MultiReader distributedlog://127.0.0.1:${ZK_PORT}/messaging/${NAMESPACE} ${SMOKESTREAM_PREFIX}1,${SMOKESTREAM_PREFIX}2,${SMOKESTREAM_PREFIX}3,${SMOKESTREAM_PREFIX}4,${SMOKESTREAM_PREFIX}5 > ${LOG_DIR}/reader.out 2>&1& +echo $! > ${LOG_DIR}/reader.pid + +# generate the records +nohup ${DLOG_ROOT}/distributedlog-tutorials/distributedlog-basic/bin/runner run com.twitter.distributedlog.basic.RecordGenerator "inet!127.0.0.1:${WP_SERVICE_PORT}" ${SMOKESTREAM_PREFIX}1 1 > ${LOG_DIR}/writer.out 2>&1& +echo $! > ${LOG_DIR}/writer.pid + +# wait for 20 seconds +sleep 20 + +# kill the writer +kill `cat ${LOG_DIR}/writer.pid` + +# stop the reader +kill `cat ${LOG_DIR}/reader.pid` + +# check the number of records received +NUM_RECORDS=`cat ${LOG_DIR}/reader.out | grep "record-" | wc -l` + +echo "Received ${NUM_RECORDS} records : " +cat ${LOG_DIR}/reader.out + +if [ $NUM_RECORDS -lt 18 ]; then + echo "[FAILURE] Received less than 18 records." + exit 1 +fi + +# stop a write proxy +${DLOG_ROOT}/distributedlog-service/bin/dlog-daemon.sh stop writeproxy + +# stop the sandbox +kill `cat ${LOG_DIR}/sandbox.pid` + +exit 0 http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/dc1bb30b/scripts/runner ---------------------------------------------------------------------- diff --git a/scripts/runner b/scripts/runner index 937829a..78ce126 100755 --- a/scripts/runner +++ b/scripts/runner @@ -23,9 +23,8 @@ set -e BASEDIR=$(dirname "$0") -DLOG_ROOT="${BASEDIR}/.." - -DLOG_HOME="${DLOG_ROOT}/distributedlog-service" +DLOG_ROOT="${BASEDIR}/../../.." +DLOG_HOME="${BASEDIR}/.." usage() { cat <<EOF @@ -45,16 +44,12 @@ These variable can also be set in conf/dlogenv.sh EOF } -cd "${DLOG_ROOT}" -source ./scripts/common.sh +source "${DLOG_ROOT}/scripts/common.sh" # get arguments COMMAND="${1}" shift -#Change to DLOG_HOME to support relative paths -cd "${DLOG_HOME}" - case "${COMMAND}" in run) java ${OPTS} $@
