Repository: kafka Updated Branches: refs/heads/trunk 0f00ec97a -> 2e91806db
KAFKA-2820: systest log level Restores control over log level in system test service class KafkaService. Author: Geoff Anderson <[email protected]> Reviewers: Ismael Juma, Ewen Cheslack-Postava Closes #538 from granders/KAFKA-2820-systest-log-level Project: http://git-wip-us.apache.org/repos/asf/kafka/repo Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/2e91806d Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/2e91806d Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/2e91806d Branch: refs/heads/trunk Commit: 2e91806db45b3854bd2fadb69cee4ecae9fd43ac Parents: 0f00ec9 Author: Geoff Anderson <[email protected]> Authored: Wed Nov 18 15:11:17 2015 -0800 Committer: Confluent <[email protected]> Committed: Wed Nov 18 15:11:17 2015 -0800 ---------------------------------------------------------------------- tests/kafkatest/services/kafka/kafka.py | 29 +++++++----- .../services/kafka/templates/log4j.properties | 46 ++++++++++---------- tests/kafkatest/services/verifiable_producer.py | 1 - 3 files changed, 40 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kafka/blob/2e91806d/tests/kafkatest/services/kafka/kafka.py ---------------------------------------------------------------------- diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py index a7a1581..15f541d 100644 --- a/tests/kafkatest/services/kafka/kafka.py +++ b/tests/kafkatest/services/kafka/kafka.py @@ -34,17 +34,22 @@ import os.path class KafkaService(JmxMixin, Service): PERSISTENT_ROOT = "/mnt" - LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "kafka-log4j.properties") + STDOUT_CAPTURE = os.path.join(PERSISTENT_ROOT, "kafka.log") + STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "kafka.log") + LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "kafka-log4j.properties") + # Logs such as controller.log, server.log, etc all go here + OPERATIONAL_LOG_DIR = os.path.join(PERSISTENT_ROOT, "kafka-operational-logs") + # Kafka log segments etc go here + DATA_LOG_DIR = os.path.join(PERSISTENT_ROOT, "kafka-data-logs") + CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "kafka.properties") + logs = { - "kafka_log": { - "path": "/mnt/kafka.log", - "collect_default": True}, "kafka_operational_logs": { - "path": "/mnt/kafka-operational-logs", + "path": OPERATIONAL_LOG_DIR, "collect_default": True}, "kafka_data": { - "path": "/mnt/kafka-data-logs", + "path": DATA_LOG_DIR, "collect_default": False} } @@ -107,24 +112,24 @@ class KafkaService(JmxMixin, Service): def start_cmd(self, node): cmd = "export JMX_PORT=%d; " % self.jmx_port - cmd += "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG_FILE - cmd += "export LOG_DIR=/mnt/kafka-operational-logs/; " + cmd += "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG + cmd += "export LOG_DIR=%s; " % KafkaService.OPERATIONAL_LOG_DIR cmd += "export KAFKA_OPTS=%s; " % self.security_config.kafka_opts - cmd += "/opt/" + kafka_dir(node) + "/bin/kafka-server-start.sh /mnt/kafka.properties 1>> /mnt/kafka.log 2>> /mnt/kafka.log &" + cmd += "/opt/" + kafka_dir(node) + "/bin/kafka-server-start.sh %s 1>> %s 2>> %s &" % (KafkaService.CONFIG_FILE, KafkaService.STDOUT_CAPTURE, KafkaService.STDERR_CAPTURE) return cmd def start_node(self, node): prop_file = self.prop_file(node) self.logger.info("kafka.properties:") self.logger.info(prop_file) - node.account.create_file("/mnt/kafka.properties", prop_file) - node.account.create_file(self.LOG4J_CONFIG_FILE, self.render('log4j.properties')) + node.account.create_file(KafkaService.CONFIG_FILE, prop_file) + node.account.create_file(self.LOG4J_CONFIG, self.render('log4j.properties', log_dir=KafkaService.OPERATIONAL_LOG_DIR)) self.security_config.setup_node(node) cmd = self.start_cmd(node) self.logger.debug("Attempting to start KafkaService on %s with command: %s" % (str(node.account), cmd)) - with node.account.monitor_log("/mnt/kafka.log") as monitor: + with node.account.monitor_log(KafkaService.STDOUT_CAPTURE) as monitor: node.account.ssh(cmd) monitor.wait_until("Kafka Server.*started", timeout_sec=30, err_msg="Kafka server didn't finish startup") http://git-wip-us.apache.org/repos/asf/kafka/blob/2e91806d/tests/kafkatest/services/kafka/templates/log4j.properties ---------------------------------------------------------------------- diff --git a/tests/kafkatest/services/kafka/templates/log4j.properties b/tests/kafkatest/services/kafka/templates/log4j.properties index bf816e7..fb38e1f 100644 --- a/tests/kafkatest/services/kafka/templates/log4j.properties +++ b/tests/kafkatest/services/kafka/templates/log4j.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=INFO, stdout +log4j.rootLogger={{ log_level }}, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout @@ -21,67 +21,67 @@ log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log +log4j.appender.kafkaAppender.File={{ log_dir }}/server.log log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log +log4j.appender.stateChangeAppender.File={{ log_dir }}/state-change.log log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log +log4j.appender.requestAppender.File={{ log_dir }}/kafka-request.log log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log +log4j.appender.cleanerAppender.File={{ log_dir }}/log-cleaner.log log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log +log4j.appender.controllerAppender.File={{ log_dir }}/controller.log log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log +log4j.appender.authorizerAppender.File={{ log_dir }}/kafka-authorizer.log log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n # Turn on all our debugging info -#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender -#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender -#log4j.logger.kafka.perf=DEBUG, kafkaAppender -#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender -#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG -log4j.logger.kafka=INFO, kafkaAppender - -log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender +log4j.logger.kafka.producer.async.DefaultEventHandler={{ log_level }}, kafkaAppender +log4j.logger.kafka.client.ClientUtils={{ log_level }}, kafkaAppender +log4j.logger.kafka.perf={{ log_level }}, kafkaAppender +log4j.logger.kafka.perf.ProducerPerformance$ProducerThread={{ log_level }}, kafkaAppender +log4j.logger.org.I0Itec.zkclient.ZkClient={{ log_level }} +log4j.logger.kafka={{ log_level }}, kafkaAppender + +log4j.logger.kafka.network.RequestChannel$={{ log_level }}, requestAppender log4j.additivity.kafka.network.RequestChannel$=false -#log4j.logger.kafka.network.Processor=TRACE, requestAppender -#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender -#log4j.additivity.kafka.server.KafkaApis=false -log4j.logger.kafka.request.logger=WARN, requestAppender +log4j.logger.kafka.network.Processor={{ log_level }}, requestAppender +log4j.logger.kafka.server.KafkaApis={{ log_level }}, requestAppender +log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.request.logger={{ log_level }}, requestAppender log4j.additivity.kafka.request.logger=false -log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.logger.kafka.controller={{ log_level }}, controllerAppender log4j.additivity.kafka.controller=false -log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender +log4j.logger.kafka.log.LogCleaner={{ log_level }}, cleanerAppender log4j.additivity.kafka.log.LogCleaner=false -log4j.logger.state.change.logger=TRACE, stateChangeAppender +log4j.logger.state.change.logger={{ log_level }}, stateChangeAppender log4j.additivity.state.change.logger=false #Change this to debug to get the actual audit log for authorizer. -log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender +log4j.logger.kafka.authorizer.logger={{ log_level }}, authorizerAppender log4j.additivity.kafka.authorizer.logger=false http://git-wip-us.apache.org/repos/asf/kafka/blob/2e91806d/tests/kafkatest/services/verifiable_producer.py ---------------------------------------------------------------------- diff --git a/tests/kafkatest/services/verifiable_producer.py b/tests/kafkatest/services/verifiable_producer.py index 9985bbd..b2d2b97 100644 --- a/tests/kafkatest/services/verifiable_producer.py +++ b/tests/kafkatest/services/verifiable_producer.py @@ -17,7 +17,6 @@ from ducktape.services.background_thread import BackgroundThreadService from kafkatest.services.kafka.directory import kafka_dir, KAFKA_TRUNK from kafkatest.services.kafka.version import TRUNK, LATEST_0_8_2 -from kafkatest.services.security.security_config import SecurityConfig import json import os
