KAFKA-3615: Exclude test jars in kafka-run-class.sh granders hachikuji Can you take a look when you have time? Appreciate your time to review.
Author: Liquan Pei <[email protected]> Reviewers: Grant Henke <[email protected]>, Geoff Anderson <[email protected]>, Ewen Cheslack-Postava <[email protected]> Closes #1263 from Ishiihara/classpath-no-test-jar Project: http://git-wip-us.apache.org/repos/asf/kafka/repo Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/eb50d2f6 Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/eb50d2f6 Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/eb50d2f6 Branch: refs/heads/0.10.0 Commit: eb50d2f6ca9dee041689db6fe397a83ece05b9e2 Parents: fe6c481 Author: Liquan Pei <[email protected]> Authored: Fri Apr 29 10:28:33 2016 -0700 Committer: Ewen Cheslack-Postava <[email protected]> Committed: Fri Apr 29 10:29:00 2016 -0700 ---------------------------------------------------------------------- bin/kafka-run-class.sh | 57 ++++++++++++++++++----- tests/kafkatest/services/security/minikdc.py | 5 +- tests/kafkatest/services/streams.py | 10 ++-- 3 files changed, 54 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kafka/blob/eb50d2f6/bin/kafka-run-class.sh ---------------------------------------------------------------------- diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh index 88d43be..e7f8d2e 100755 --- a/bin/kafka-run-class.sh +++ b/bin/kafka-run-class.sh @@ -20,6 +20,24 @@ then exit 1 fi +if [ -z "$INCLUDE_TEST_JARS" ]; then + INCLUDE_TEST_JARS=false +fi + +# Exclude jars not necessary for running commands. +regex="(-(test|src|scaladoc|javadoc)\.jar|jar.asc)$" +should_include_file() { + if [ "$INCLUDE_TEST_JARS" = true ]; then + return 0 + fi + file=$1 + if [ -z "$(echo "$file" | egrep "$regex")" ] ; then + return 0 + else + return 1 + fi +} + base_dir=$(dirname $0)/.. if [ -z "$SCALA_VERSION" ]; then @@ -41,24 +59,32 @@ do fi done -for file in $base_dir/examples/build/libs//kafka-examples*.jar; +for file in $base_dir/examples/build/libs/kafka-examples*.jar; do - CLASSPATH=$CLASSPATH:$file + if should_include_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done for file in $base_dir/clients/build/libs/kafka-clients*.jar; do - CLASSPATH=$CLASSPATH:$file + if should_include_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done for file in $base_dir/streams/build/libs/kafka-streams*.jar; do - CLASSPATH=$CLASSPATH:$file + if should_include_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done for file in $base_dir/streams/examples/build/libs/kafka-streams-examples*.jar; do - CLASSPATH=$CLASSPATH:$file + if should_include_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done for file in $base_dir/streams/build/dependant-libs-${SCALA_VERSION}/rocksdb*.jar; @@ -68,7 +94,9 @@ done for file in $base_dir/tools/build/libs/kafka-tools*.jar; do - CLASSPATH=$CLASSPATH:$file + if should_include_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done for dir in $base_dir/tools/build/dependant-libs-${SCALA_VERSION}*; @@ -80,7 +108,9 @@ for cc_pkg in "api" "runtime" "file" "json" "tools" do for file in $base_dir/connect/${cc_pkg}/build/libs/connect-${cc_pkg}*.jar; do - CLASSPATH=$CLASSPATH:$file + if should_include_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done if [ -d "$base_dir/connect/${cc_pkg}/build/dependant-libs" ] ; then CLASSPATH=$CLASSPATH:$base_dir/connect/${cc_pkg}/build/dependant-libs/* @@ -88,11 +118,18 @@ do done # classpath addition for release -CLASSPATH=$CLASSPATH:$base_dir/libs/* +for file in $base_dir/libs; +do + if should_include_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi +done for file in $base_dir/core/build/libs/kafka_${SCALA_BINARY_VERSION}*.jar; do - CLASSPATH=$CLASSPATH:$file + if should_include_file "$file"; then + CLASSPATH=$CLASSPATH:$file + fi done shopt -u nullglob @@ -145,8 +182,6 @@ if [ "x$KAFKA_DEBUG" != "x" ]; then JAVA_DEBUG_OPTS="$DEFAULT_JAVA_DEBUG_OPTS" fi - - echo "Enabling Java debug options: $JAVA_DEBUG_OPTS" KAFKA_OPTS="$JAVA_DEBUG_OPTS $KAFKA_OPTS" fi http://git-wip-us.apache.org/repos/asf/kafka/blob/eb50d2f6/tests/kafkatest/services/security/minikdc.py ---------------------------------------------------------------------- diff --git a/tests/kafkatest/services/security/minikdc.py b/tests/kafkatest/services/security/minikdc.py index b376e26..d83aede 100644 --- a/tests/kafkatest/services/security/minikdc.py +++ b/tests/kafkatest/services/security/minikdc.py @@ -23,6 +23,7 @@ from os import remove, close from io import open import uuid + class MiniKdc(Service): logs = { @@ -54,9 +55,7 @@ class MiniKdc(Service): remove(file_path) move(abs_path, file_path) - def start_node(self, node): - node.account.ssh("mkdir -p %s" % MiniKdc.WORK_DIR, allow_fail=False) props_file = self.render('minikdc.properties', node=node) node.account.create_file(MiniKdc.PROPS_FILE, props_file) @@ -69,7 +68,7 @@ class MiniKdc(Service): jar_paths = self.core_jar_paths(node, "dependant-testlibs") + self.core_jar_paths(node, "libs") classpath = ":".join(jar_paths) - cmd = "CLASSPATH=%s /opt/%s/bin/kafka-run-class.sh kafka.security.minikdc.MiniKdc %s %s %s %s 1>> %s 2>> %s &" % (classpath, kafka_dir(node), MiniKdc.WORK_DIR, MiniKdc.PROPS_FILE, MiniKdc.KEYTAB_FILE, principals, MiniKdc.LOG_FILE, MiniKdc.LOG_FILE) + cmd = "INCLUDE_TEST_JARS=true CLASSPATH=%s /opt/%s/bin/kafka-run-class.sh kafka.security.minikdc.MiniKdc %s %s %s %s 1>> %s 2>> %s &" % (classpath, kafka_dir(node), MiniKdc.WORK_DIR, MiniKdc.PROPS_FILE, MiniKdc.KEYTAB_FILE, principals, MiniKdc.LOG_FILE, MiniKdc.LOG_FILE) self.logger.debug("Attempting to start MiniKdc on %s with command: %s" % (str(node.account), cmd)) with node.account.monitor_log(MiniKdc.LOG_FILE) as monitor: node.account.ssh(cmd) http://git-wip-us.apache.org/repos/asf/kafka/blob/eb50d2f6/tests/kafkatest/services/streams.py ---------------------------------------------------------------------- diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py index dcbcc69..53d967e 100644 --- a/tests/kafkatest/services/streams.py +++ b/tests/kafkatest/services/streams.py @@ -15,10 +15,11 @@ from ducktape.services.service import Service from ducktape.utils.util import wait_until -from ducktape.errors import DucktapeError from kafkatest.services.kafka.directory import kafka_dir -import signal, random, requests, os.path, json +import signal +import os.path + class StreamsSmokeTestBaseService(Service): """Base class for Streams Smoke Test services providing some common settings and functionality""" @@ -46,7 +47,7 @@ class StreamsSmokeTestBaseService(Service): def __init__(self, context, kafka, command): super(StreamsSmokeTestBaseService, self).__init__(context, 1) self.kafka = kafka - self.args = { 'command': command } + self.args = {'command': command} @property def node(self): @@ -107,7 +108,7 @@ class StreamsSmokeTestBaseService(Service): args['kafka_dir'] = kafka_dir(node) cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ - "/opt/%(kafka_dir)s/bin/kafka-run-class.sh org.apache.kafka.streams.smoketest.StreamsSmokeTest " \ + "INCLUDE_TEST_JARS=true /opt/%(kafka_dir)s/bin/kafka-run-class.sh org.apache.kafka.streams.smoketest.StreamsSmokeTest " \ " %(command)s %(kafka)s %(zk)s %(state_dir)s " \ " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args @@ -131,6 +132,7 @@ class StreamsSmokeTestDriverService(StreamsSmokeTestBaseService): def __init__(self, context, kafka): super(StreamsSmokeTestDriverService, self).__init__(context, kafka, "run") + class StreamsSmokeTestJobRunnerService(StreamsSmokeTestBaseService): def __init__(self, context, kafka): super(StreamsSmokeTestJobRunnerService, self).__init__(context, kafka, "process")
