Repository: storm Updated Branches: refs/heads/0.10.x-branch bbcae65d3 -> 4ad25ea32
Merge branch 'log4j-changes' of https://github.com/kishorvpatil/incubator-storm into STORM-833 STORM-833: Logging framework logback -> log4j 2.x Conflicts: bin/storm-config.cmd Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ce60f3b5 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ce60f3b5 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ce60f3b5 Branch: refs/heads/0.10.x-branch Commit: ce60f3b5a2686c03d9bf3ff19c3bb56bee9f90d3 Parents: bbcae65 Author: Robert (Bobby) Evans <[email protected]> Authored: Thu Jun 4 13:01:54 2015 -0500 Committer: Robert (Bobby) Evans <[email protected]> Committed: Thu Jun 4 13:10:53 2015 -0500 ---------------------------------------------------------------------- bin/storm-config.cmd | 4 +- bin/storm.py | 33 +++++--- conf/log4j2.xml | 32 ++++++++ conf/logback.xml | 28 ------- log4j2/cluster.xml | 76 +++++++++++++++++ log4j2/worker.xml | 77 ++++++++++++++++++ logback/cluster.xml | 85 -------------------- logback/worker.xml | 41 ---------- pom.xml | 25 ++++-- storm-core/pom.xml | 12 ++- .../src/clj/backtype/storm/daemon/logviewer.clj | 11 +-- .../src/clj/backtype/storm/daemon/nimbus.clj | 5 +- .../clj/backtype/storm/daemon/supervisor.clj | 14 +++- .../src/clj/backtype/storm/daemon/worker.clj | 2 +- storm-core/src/clj/backtype/storm/util.clj | 9 ++- storm-core/src/jvm/backtype/storm/Config.java | 11 +++ .../src/jvm/backtype/storm/LogWriter.java | 83 +++++++++++++++++++ .../test/clj/backtype/storm/logviewer_test.clj | 8 +- .../test/clj/backtype/storm/supervisor_test.clj | 30 ++++++- storm-dist/binary/src/main/assembly/binary.xml | 8 +- 20 files changed, 398 insertions(+), 196 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/bin/storm-config.cmd ---------------------------------------------------------------------- diff --git a/bin/storm-config.cmd b/bin/storm-config.cmd index 5203241..7f88280 100644 --- a/bin/storm-config.cmd +++ b/bin/storm-config.cmd @@ -110,7 +110,7 @@ if not %STORM_LOGBACK_CONFIGURATION_DIR% == nil ( @rem if not defined STORM_LOGBACK_CONFIGURATION_FILE ( - set STORM_LOGBACK_CONFIGURATION_FILE=%STORM_HOME%\logback\cluster.xml + set STORM_LOGBACK_CONFIGURATION_FILE=%STORM_HOME%\log4j2\cluster.xml ) "%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%" backtype.storm.command.config_value java.library.path > temp.txt @@ -126,7 +126,7 @@ FOR /F "delims=" %%i in (temp.txt) do ( :storm_opts set STORM_OPTS=-Dstorm.options= -Dstorm.home=%STORM_HOME% -Djava.library.path=%JAVA_LIBRARY_PATH% - set STORM_OPTS=%STORM_OPTS% -Dlogback.configurationFile=%STORM_LOGBACK_CONFIGURATION_FILE% + set STORM_OPTS=%STORM_OPTS% -Dlog4j.configurationFile=%STORM_LOGBACK_CONFIGURATION_FILE% set STORM_OPTS=%STORM_OPTS% -Dstorm.log.dir=%STORM_LOG_DIR% del /F temp.txt http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/bin/storm.py ---------------------------------------------------------------------- diff --git a/bin/storm.py b/bin/storm.py index 455a4c0..ba89304 100755 --- a/bin/storm.py +++ b/bin/storm.py @@ -76,7 +76,7 @@ if (not os.path.isfile(os.path.join(USER_CONF_DIR, "storm.yaml"))): STORM_LIB_DIR = os.path.join(STORM_DIR, "lib") STORM_BIN_DIR = os.path.join(STORM_DIR, "bin") -STORM_LOGBACK_CONF_DIR = os.path.join(STORM_DIR, "logback") +STORM_LOG4J_CONF_DIR = os.path.join(STORM_DIR, "log4j2") init_storm_env() @@ -177,13 +177,15 @@ def parse_args(string): args = [re.compile(r"'((?:[^'\\]|\\.)*)'").sub('\\1', x) for x in args] return [re.compile(r'\\(.)').sub('\\1', x) for x in args] -def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[], fork=False, daemon=True): +def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[], fork=False, daemon=True, daemonName=""): global CONFFILE storm_log_dir = confvalue("storm.log.dir",[CLUSTER_CONF_DIR]) if(storm_log_dir == None or storm_log_dir == "nil"): storm_log_dir = os.path.join(STORM_DIR, "logs") all_args = [ - "java", jvmtype, get_config_opts(), + JAVA_CMD, jvmtype, + "-Ddaemon.name=" + daemonName, + get_config_opts(), "-Dstorm.home=" + STORM_DIR, "-Dstorm.log.dir=" + storm_log_dir, "-Djava.library.path=" + confvalue("java.library.path", extrajars, daemon), @@ -326,12 +328,12 @@ def repl(): cppaths = [CLUSTER_CONF_DIR] exec_storm_class("clojure.main", jvmtype="-client", extrajars=cppaths) -def get_logback_conf_dir(): +def get_log4j_conf_dir(): cppaths = [CLUSTER_CONF_DIR] - storm_logback_conf_dir = confvalue("storm.logback.conf.dir", cppaths) - if(storm_logback_conf_dir == None or storm_logback_conf_dir == "nil"): - storm_logback_conf_dir = STORM_LOGBACK_CONF_DIR - return storm_logback_conf_dir + storm_log4j_conf_dir = confvalue("storm.logback.conf.dir", cppaths) + if(storm_log4j_conf_dir == None or storm_log4j_conf_dir == "nil"): + storm_log4j_conf_dir = STORM_LOG4J_CONF_DIR + return storm_log4j_conf_dir def nimbus(klass="backtype.storm.daemon.nimbus"): """Syntax: [storm nimbus] @@ -345,11 +347,12 @@ def nimbus(klass="backtype.storm.daemon.nimbus"): cppaths = [CLUSTER_CONF_DIR] jvmopts = parse_args(confvalue("nimbus.childopts", cppaths)) + [ "-Dlogfile.name=nimbus.log", - "-Dlogback.configurationFile=" + os.path.join(get_logback_conf_dir(), "cluster.xml"), + "-Dlog4j.configurationFile=" + os.path.join(get_log4j_conf_dir(), "cluster.xml"), ] exec_storm_class( klass, jvmtype="-server", + daemonName="nimbus", extrajars=cppaths, jvmopts=jvmopts) @@ -365,11 +368,12 @@ def supervisor(klass="backtype.storm.daemon.supervisor"): cppaths = [CLUSTER_CONF_DIR] jvmopts = parse_args(confvalue("supervisor.childopts", cppaths)) + [ "-Dlogfile.name=supervisor.log", - "-Dlogback.configurationFile=" + os.path.join(get_logback_conf_dir(), "cluster.xml"), + "-Dlog4j.configurationFile=" + os.path.join(get_log4j_conf_dir(), "cluster.xml"), ] exec_storm_class( klass, jvmtype="-server", + daemonName="supervisor", extrajars=cppaths, jvmopts=jvmopts) @@ -386,11 +390,12 @@ def ui(): cppaths = [CLUSTER_CONF_DIR] jvmopts = parse_args(confvalue("ui.childopts", cppaths)) + [ "-Dlogfile.name=ui.log", - "-Dlogback.configurationFile=" + os.path.join(get_logback_conf_dir(), "cluster.xml") + "-Dlog4j.configurationFile=" + os.path.join(get_log4j_conf_dir(), "cluster.xml") ] exec_storm_class( "backtype.storm.ui.core", jvmtype="-server", + daemonName="ui", jvmopts=jvmopts, extrajars=[STORM_DIR, CLUSTER_CONF_DIR]) @@ -407,11 +412,12 @@ def logviewer(): cppaths = [CLUSTER_CONF_DIR] jvmopts = parse_args(confvalue("logviewer.childopts", cppaths)) + [ "-Dlogfile.name=logviewer.log", - "-Dlogback.configurationFile=" + os.path.join(get_logback_conf_dir(), "cluster.xml") + "-Dlog4j.configurationFile=" + os.path.join(get_log4j_conf_dir(), "cluster.xml") ] exec_storm_class( "backtype.storm.daemon.logviewer", jvmtype="-server", + daemonName="logviewer", jvmopts=jvmopts, extrajars=[STORM_DIR, CLUSTER_CONF_DIR]) @@ -427,11 +433,12 @@ def drpc(): cppaths = [CLUSTER_CONF_DIR] jvmopts = parse_args(confvalue("drpc.childopts", cppaths)) + [ "-Dlogfile.name=drpc.log", - "-Dlogback.configurationFile=" + os.path.join(get_logback_conf_dir(), "cluster.xml") + "-Dlog4j.configurationFile=" + os.path.join(get_log4j_conf_dir(), "cluster.xml") ] exec_storm_class( "backtype.storm.daemon.drpc", jvmtype="-server", + daemonName="drpc", jvmopts=jvmopts, extrajars=[CLUSTER_CONF_DIR]) http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/conf/log4j2.xml ---------------------------------------------------------------------- diff --git a/conf/log4j2.xml b/conf/log4j2.xml new file mode 100644 index 0000000..cfc8330 --- /dev/null +++ b/conf/log4j2.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration monitorInterval="60"> + <Appenders> + <Console name="Console" target="SYSTEM_OUT"> + <PatternLayout pattern="%-4r [%t] %-5p %c{1.} - %msg%n"/> + </Console> + </Appenders> + <Loggers> + <Logger name="org.apache.zookeeper" level="WARN"/> + <Root level="info"> + <AppenderRef ref="Console"/> + </Root> + </Loggers> +</configuration> + http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/conf/logback.xml ---------------------------------------------------------------------- diff --git a/conf/logback.xml b/conf/logback.xml deleted file mode 100644 index 7ccaae6..0000000 --- a/conf/logback.xml +++ /dev/null @@ -1,28 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<configuration scan="true" scanPeriod="30 seconds"> - <appender name="A1" class="ch.qos.logback.core.ConsoleAppender"> - <encoder> - <pattern>%-4r [%t] %-5p %c - %m%n</pattern> - </encoder> - </appender> - <logger name="org.apache.zookeeper" level="WARN"/> - <root level="INFO"> - <appender-ref ref="A1"/> - </root> -</configuration> http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/log4j2/cluster.xml ---------------------------------------------------------------------- diff --git a/log4j2/cluster.xml b/log4j2/cluster.xml new file mode 100644 index 0000000..2301e21 --- /dev/null +++ b/log4j2/cluster.xml @@ -0,0 +1,76 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration monitorInterval="60"> +<properties> + <property name="pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} [%p] %msg%n</property> + <property name="patternMetris">%d %-8r %m%n</property> +</properties> +<appenders> + <RollingFile name="A1" + fileName="${sys:storm.home}/logs/${sys:logfile.name}" + filePattern="${sys:storm.home}/logs/${sys:logfile.name}.%i"> + <PatternLayout> + <pattern>${pattern}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <DefaultRolloverStrategy max="9"/> + </Policies> + </RollingFile> + <RollingFile name="ACCESS" + fileName="${sys:storm.home}/logs/access.log" + filePattern="${sys:storm.home}/logs/access.log.%i"> + <PatternLayout> + <pattern>${pattern}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <DefaultRolloverStrategy max="9"/> + </Policies> + </RollingFile> + <RollingFile name="METRICS" + fileName="${sys:storm.home}/logs/metrics.log" + filePattern="${sys:storm.home}/logs/metrics.log.%i"> + <PatternLayout> + <pattern>${patternMetris}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="2 MB"/> <!-- Or every 100 MB --> + <DefaultRolloverStrategy max="9"/> + </Policies> + </RollingFile> + <Syslog name="syslog" format="RFC5424" host="localhost" port="514" + protocol="UDP" appName="[${sys:daemon.name}]" mdcId="mdc" includeMDC="true" + facility="LOCAL5" enterpriseNumber="18060" newLine="true" exceptionPattern="%rEx{full}" + messageId="[${sys:user.name}:S0]" id="storm"/> +</appenders> +<loggers> + + <Logger name="backtype.storm.security.auth.authorizer" level="info"> + <AppenderRef ref="ACCESS"/> + </Logger> + <Logger name="backtype.storm.metric.LoggingMetricsConsumer" level="info"> + <AppenderRef ref="METRICS"/> + </Logger> + <root level="info"> <!-- We log everything --> + <appender-ref ref="A1"/> + <appender-ref ref="syslog"/> + </root> +</loggers> +</configuration> http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/log4j2/worker.xml ---------------------------------------------------------------------- diff --git a/log4j2/worker.xml b/log4j2/worker.xml new file mode 100644 index 0000000..54d612c --- /dev/null +++ b/log4j2/worker.xml @@ -0,0 +1,77 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration monitorInterval="60"> +<properties> + <property name="pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} [%p] %msg%n</property> + <property name="patternNoTime">%msg%n</property> +</properties> +<appenders> + <RollingFile name="A1" + fileName="${sys:storm.home}/logs/${sys:logfile.name}" + filePattern="${sys:storm.home}/logs/${sys:logfile.name}.%i.gz"> + <PatternLayout> + <pattern>${pattern}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <DefaultRolloverStrategy max="9"/> + </Policies> + </RollingFile> + <RollingFile name="STDOUT" + fileName="${sys:storm.home}/logs/${sys:logfile.name}.out" + filePattern="${sys:storm.home}/logs/${sys:logfile.name}.out.%i.gz"> + <PatternLayout> + <pattern>${patternNoTime}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <DefaultRolloverStrategy max="4"/> + </Policies> + </RollingFile> + <RollingFile name="STDERR" + fileName="${sys:storm.home}/logs/${sys:logfile.name}.err" + filePattern="${sys:storm.home}/logs/${sys:logfile.name}.err.%i.gz"> + <PatternLayout> + <pattern>${patternNoTime}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + <DefaultRolloverStrategy max="4"/> + </Policies> + </RollingFile> + <Syslog name="syslog" format="RFC5424" host="localhost" port="514" + protocol="UDP" appName="[${sys:storm.id}:${sys:worker.port}]" mdcId="mdc" includeMDC="true" + facility="LOCAL5" enterpriseNumber="18060" newLine="true" exceptionPattern="%rEx{full}" + messageId="[${sys:user.name}:${sys:logging.sensitivity}]" id="storm"/> +</appenders> +<loggers> + <root level="info"> <!-- We log everything --> + <appender-ref ref="A1"/> + <appender-ref ref="syslog"/> + </root> + <Logger name="STDERR" level="INFO"> + <appender-ref ref="STDERR"/> + <appender-ref ref="syslog"/> + </Logger> + <Logger name="STDOUT" level="INFO"> + <appender-ref ref="STDOUT"/> + <appender-ref ref="syslog"/> + </Logger> +</loggers> +</configuration> http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/logback/cluster.xml ---------------------------------------------------------------------- diff --git a/logback/cluster.xml b/logback/cluster.xml deleted file mode 100644 index 487e119..0000000 --- a/logback/cluster.xml +++ /dev/null @@ -1,85 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<configuration scan="true" scanPeriod="60 seconds"> - <appender name="A1" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${storm.log.dir}/${logfile.name}</file> - <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> - <fileNamePattern>${storm.log.dir}/${logfile.name}.%i</fileNamePattern> - <minIndex>1</minIndex> - <maxIndex>9</maxIndex> - </rollingPolicy> - - <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> - <maxFileSize>100MB</maxFileSize> - </triggeringPolicy> - - <encoder> - <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSZZ} %c{1} [%p] %m%n</pattern> - </encoder> - </appender> - - <appender name="ACCESS" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${storm.log.dir}/access.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> - <fileNamePattern>${storm.log.dir}/access.log.%i</fileNamePattern> - <minIndex>1</minIndex> - <maxIndex>9</maxIndex> - </rollingPolicy> - - <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> - <maxFileSize>100MB</maxFileSize> - </triggeringPolicy> - - <encoder> - <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSZZ} %c{1} [%p] %m%n</pattern> - </encoder> - </appender> - - <appender name="METRICS" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${storm.log.dir}/metrics.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> - <fileNamePattern>${storm.log.dir}/metrics.log.%i</fileNamePattern> - <minIndex>1</minIndex> - <maxIndex>9</maxIndex> - </rollingPolicy> - - <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> - <maxFileSize>2MB</maxFileSize> - </triggeringPolicy> - - <encoder> - <pattern>%d %-8r %m%n</pattern> - </encoder> - </appender> - - <root level="INFO"> - <appender-ref ref="A1"/> - </root> - - <logger name="backtype.storm.security.auth.authorizer" additivity="false"> - <level value="INFO" /> - <appender-ref ref="ACCESS" /> - </logger> - - <logger name="backtype.storm.metric.LoggingMetricsConsumer" additivity="false" > - <level value="INFO"/> - <appender-ref ref="METRICS"/> - </logger> - -</configuration> http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/logback/worker.xml ---------------------------------------------------------------------- diff --git a/logback/worker.xml b/logback/worker.xml deleted file mode 100644 index 3d9009e..0000000 --- a/logback/worker.xml +++ /dev/null @@ -1,41 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<configuration scan="true" scanPeriod="60 seconds"> - <appender name="A1" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${storm.log.dir}/${logfile.name}</file> - <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> - <fileNamePattern>${storm.log.dir}/${logfile.name}.%i</fileNamePattern> - <minIndex>1</minIndex> - <maxIndex>9</maxIndex> - </rollingPolicy> - - <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> - <maxFileSize>100MB</maxFileSize> - </triggeringPolicy> - - <encoder> - <pattern>%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n</pattern> - </encoder> - </appender> - - <root level="INFO"> - <appender-ref ref="A1"/> - </root> - -</configuration> http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 6f2967b..0ba9851 100644 --- a/pom.xml +++ b/pom.xml @@ -209,9 +209,9 @@ <disruptor.version>2.10.1</disruptor.version> <jgrapht.version>0.9.0</jgrapht.version> <guava.version>16.0.1</guava.version> - <logback-classic.version>1.0.13</logback-classic.version> - <log4j-over-slf4j.version>1.6.6</log4j-over-slf4j.version> <netty.version>3.9.0.Final</netty.version> + <log4j-over-slf4j.version>1.6.6</log4j-over-slf4j.version> + <log4j.version>2.1</log4j.version> <clojure.tools.nrepl.version>0.2.3</clojure.tools.nrepl.version> <clojure-complete.version>0.2.3</clojure-complete.version> <mockito.version>1.9.5</mockito.version> @@ -468,9 +468,24 @@ <version>${guava.version}</version> </dependency> <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - <version>${logback-classic.version}</version> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <version>${log4j.version}</version> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <version>${log4j.version}</version> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <version>${log4j.version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>${log4j-over-slf4j.version}</version> </dependency> <dependency> <groupId>org.slf4j</groupId> http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-core/pom.xml ---------------------------------------------------------------------- diff --git a/storm-core/pom.xml b/storm-core/pom.xml index 6dad8ac..178e751 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -200,8 +200,16 @@ <artifactId>guava</artifactId> </dependency> <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> </dependency> <dependency> <groupId>org.slf4j</groupId> http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-core/src/clj/backtype/storm/daemon/logviewer.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj index 1c24440..8a38df6 100644 --- a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj +++ b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj @@ -21,9 +21,10 @@ (:use [backtype.storm config util log timer]) (:use [backtype.storm.ui helpers]) (:import [org.slf4j LoggerFactory]) - (:import [ch.qos.logback.classic Logger]) - (:import [ch.qos.logback.core FileAppender]) (:import [java.io File FileFilter FileInputStream]) + (:import [org.apache.logging.log4j LogManager]) + (:import [org.apache.logging.log4j.core Appender LoggerContext]) + (:import [org.apache.logging.log4j.core.appender RollingFileAppender]) (:import [org.yaml.snakeyaml Yaml] [org.yaml.snakeyaml.constructor SafeConstructor]) (:import [backtype.storm.ui InvalidRequestException] @@ -210,9 +211,9 @@ Note that if anything goes wrong, this will throw an Error and exit." [appender-name] - (let [appender (.getAppender (LoggerFactory/getLogger Logger/ROOT_LOGGER_NAME) appender-name)] - (if (and appender-name appender (instance? FileAppender appender)) - (.getParent (File. (.getFile appender))) + (let [appender (.getAppender (.getConfiguration (LogManager/getContext)) appender-name)] + (if (and appender-name appender (instance? RollingFileAppender appender)) + (.getParent (File. (.getFileName appender))) (throw (RuntimeException. "Log viewer could not find configured appender, or the appender is not a FileAppender. Please check that the appender name configured in storm and logback agree."))))) http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-core/src/clj/backtype/storm/daemon/nimbus.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj index e079ba4..f23d9b1 100644 --- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj +++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj @@ -1089,7 +1089,10 @@ (when (and (Utils/isZkAuthenticationConfiguredStormServer conf) (not (Utils/isZkAuthenticationConfiguredTopology storm-conf))) (throw (IllegalArgumentException. "The cluster is configured for zookeeper authentication, but no payload was provided."))) - (log-message "Received topology submission for " storm-name " with conf " storm-conf) + (log-message "Received topology submission for " + storm-name + " with conf " + (redact-value storm-conf STORM-ZOOKEEPER-TOPOLOGY-AUTH-PAYLOAD)) ;; lock protects against multiple topologies being submitted at once and ;; cleanup thread killing topology in b/w assignment and starting the topology (locking (:submit-lock nimbus) http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-core/src/clj/backtype/storm/daemon/supervisor.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj index 057d2bf..ba83a7e 100644 --- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj @@ -660,7 +660,7 @@ storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home file-path-separator "logs")) storm-conf (read-storm-config) storm-log-conf-dir (storm-conf "storm.logback.conf.dir") - storm-logback-conf-dir (or storm-log-conf-dir (str storm-home file-path-separator "logback")) + storm-logback-conf-dir (or storm-log-conf-dir (str storm-home file-path-separator "log4j2")) stormroot (supervisor-stormdist-root conf storm-id) jlp (jlp stormroot conf) stormjar (supervisor-stormjar-path stormroot) @@ -674,6 +674,7 @@ top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS) gc-opts (substitute-childopts (if top-gc-opts top-gc-opts (conf WORKER-GC-CHILDOPTS)) worker-id storm-id port) user (storm-conf TOPOLOGY-SUBMITTER-USER) + logging-sensitivity (storm-conf TOPOLOGY-LOGGING-SENSITIVITY "S3") logfilename (logs-filename storm-id port) worker-childopts (when-let [s (conf WORKER-CHILDOPTS)] (substitute-childopts s worker-id storm-id port)) @@ -683,6 +684,14 @@ (merge env {"LD_LIBRARY_PATH" jlp}) {"LD_LIBRARY_PATH" jlp}) command (concat + [(java-cmd) "-cp" classpath + (str "-Dlogfile.name=" logfilename) + (str "-Dstorm.home=" storm-home) + (str "-Dstorm.id=" storm-id) + (str "-Dworker.id=" worker-id) + (str "-Dworker.port=" port) + (str "-Dlog4j.configurationFile=" storm-logback-conf-dir file-path-separator "worker.xml") + "backtype.storm.LogWriter"] [(java-cmd) "-server"] worker-childopts topo-worker-childopts @@ -693,7 +702,8 @@ (str "-Dstorm.conf.file=" storm-conf-file) (str "-Dstorm.options=" storm-options) (str "-Dstorm.log.dir=" storm-log-dir) - (str "-Dlogback.configurationFile=" storm-logback-conf-dir file-path-separator "worker.xml") + (str "-Dlogging.sensitivity=" logging-sensitivity) + (str "-Dlog4j.configurationFile=" storm-logback-conf-dir file-path-separator "worker.xml") (str "-Dstorm.id=" storm-id) (str "-Dworker.id=" worker-id) (str "-Dworker.port=" port) http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-core/src/clj/backtype/storm/daemon/worker.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/daemon/worker.clj b/storm-core/src/clj/backtype/storm/daemon/worker.clj index ae7b3ae..7157cf7 100644 --- a/storm-core/src/clj/backtype/storm/daemon/worker.clj +++ b/storm-core/src/clj/backtype/storm/daemon/worker.clj @@ -520,7 +520,7 @@ (schedule-recurring (:refresh-connections-timer worker) 0 (conf TASK-REFRESH-POLL-SECS) refresh-connections) (schedule-recurring (:refresh-active-timer worker) 0 (conf TASK-REFRESH-POLL-SECS) (partial refresh-storm-active worker)) - (log-message "Worker has topology config " (:storm-conf worker)) + (log-message "Worker has topology config " (redact-value (:storm-conf worker) STORM-ZOOKEEPER-TOPOLOGY-AUTH-PAYLOAD)) (log-message "Worker " worker-id " for storm " storm-id " on " assignment-id ":" port " has finished loading") ret )))))) http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-core/src/clj/backtype/storm/util.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj index 70e166a..3fd9f2a 100644 --- a/storm-core/src/clj/backtype/storm/util.clj +++ b/storm-core/src/clj/backtype/storm/util.clj @@ -1026,7 +1026,7 @@ (defn logs-metadata-filename [storm-id port] (str (logs-rootname storm-id port) ".yaml")) -(def worker-log-filename-pattern #"((.*-\d+-\d+)-worker-(\d+)).log") +(def worker-log-filename-pattern #"^((.*-\d+-\d+)-worker-(\d+))\.log") (defn get-log-metadata-file ([fname] @@ -1058,3 +1058,10 @@ (log-error err "Received error in main thread.. terminating server...") (.exit (Runtime/getRuntime) -2)))))))) +(defn redact-value + "Hides value for k in coll for printing coll safely" + [coll k] + (if (contains? coll k) + (assoc coll k (apply str (repeat (count (coll k)) "#"))) + coll)) + http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-core/src/jvm/backtype/storm/Config.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java index b24392f..4e15352 100644 --- a/storm-core/src/jvm/backtype/storm/Config.java +++ b/storm-core/src/jvm/backtype/storm/Config.java @@ -1298,6 +1298,17 @@ public class Config extends HashMap<String, Object> { public static final Object TOPOLOGY_SHELLBOLT_MAX_PENDING_SCHEMA = ConfigValidation.IntegerValidator; /** + * Topology central logging sensitivity to determine who has access to logs in central logging system. + * The possible values are: + * S0 - Public (open to all users on grid) + * S1 - Restricted + * S2 - Confidential + * S3 - Secret (default.) + */ + public static final String TOPOLOGY_LOGGING_SENSITIVITY="topology.logging.sensitivity"; + public static final Object TOPOLOGY_LOGGING_SENSITIVITY_SCHEMA = String.class; + + /** * The root directory in ZooKeeper for metadata about TransactionalSpouts. */ public static final String TRANSACTIONAL_ZOOKEEPER_ROOT="transactional.zookeeper.root"; http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-core/src/jvm/backtype/storm/LogWriter.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/LogWriter.java b/storm-core/src/jvm/backtype/storm/LogWriter.java new file mode 100644 index 0000000..b0857e8 --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/LogWriter.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package backtype.storm; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.InputStream; +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Launch a sub process and write files out to logs. + */ +public class LogWriter extends Thread { + private Logger logger; + private BufferedReader in; + + public LogWriter(InputStream in, Logger logger) { + this.in = new BufferedReader(new InputStreamReader(in)); + this.logger = logger; + } + + public void run() { + Logger logger = this.logger; + BufferedReader in = this.in; + String line = null; + try { + while ((line = in.readLine()) != null) { + logger.info(line); + } + } catch (IOException e) { + logger.error("Internal ERROR", e); + } finally { + try { + in.close(); + } catch (IOException e) { + logger.error("Internal ERROR", e); + } + } + } + + public void close() throws Exception { + this.join(); + } + + public static void main(String [] args) throws Exception { + ProcessBuilder pb = new ProcessBuilder(args); + Process p = pb.start(); + LogWriter err = null; + LogWriter in = null; + int ret = -1; + try { + Logger logger = LoggerFactory.getLogger("STDERR"); + err = new LogWriter(p.getErrorStream(), logger); + err.start(); + in = new LogWriter(p.getInputStream(), logger); + in.start(); + ret = p.waitFor(); + } finally { + if (err != null) err.close(); + if (in != null) in.close(); + } + System.exit(ret); + } +} http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-core/test/clj/backtype/storm/logviewer_test.clj ---------------------------------------------------------------------- diff --git a/storm-core/test/clj/backtype/storm/logviewer_test.clj b/storm-core/test/clj/backtype/storm/logviewer_test.clj index 23fa0b9..a790ddf 100644 --- a/storm-core/test/clj/backtype/storm/logviewer_test.clj +++ b/storm-core/test/clj/backtype/storm/logviewer_test.clj @@ -91,10 +91,14 @@ (deftest test-get-log-root->files-map (testing "returns map of root name to list of files" (let [files (vec (map #(java.io.File. %) ["log-1-2-worker-3.log" + "log-1-2-worker-3.log.1.gz" + "log-1-2-worker-3.log.err" + "log-1-2-worker-3.log.out" + "log-1-2-worker-3.log.out.1.gz" "log-1-2-worker-3.log.1" "log-2-4-worker-6.log.1"])) - expected {"log-1-2-worker-3" #{(files 0) (files 1)} - "log-2-4-worker-6" #{(files 2)}}] + expected {"log-1-2-worker-3" #{(files 0) (files 1) (files 2) (files 3) (files 4) (files 5)} + "log-2-4-worker-6" #{(files 6)}}] (is (= expected (logviewer/get-log-root->files-map files)))))) (deftest test-identify-worker-log-files http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-core/test/clj/backtype/storm/supervisor_test.clj ---------------------------------------------------------------------- diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj index 4c029dd..dcb8dab 100644 --- a/storm-core/test/clj/backtype/storm/supervisor_test.clj +++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj @@ -252,8 +252,18 @@ mock-storm-id "fake-storm-id" mock-worker-id "fake-worker-id" mock-cp (str file-path-separator "base" class-path-separator file-path-separator "stormjar.jar") + mock-sensitivity "S3" + mock-cp "/base:/stormjar.jar" exp-args-fn (fn [opts topo-opts classpath] - (concat [(supervisor/java-cmd) "-server"] + (concat [(supervisor/java-cmd) "-cp" classpath + (str "-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log") + "-Dstorm.home=" + (str "-Dstorm.id=" mock-storm-id) + (str "-Dworker.id=" mock-worker-id) + (str "-Dworker.port=" mock-port) + "-Dlog4j.configurationFile=/log4j2/worker.xml" + "backtype.storm.LogWriter"] + [(supervisor/java-cmd) "-server"] opts topo-opts ["-Djava.library.path=" @@ -262,7 +272,8 @@ "-Dstorm.conf.file=" "-Dstorm.options=" (str "-Dstorm.log.dir=" file-path-separator "logs") - (str "-Dlogback.configurationFile=" file-path-separator "logback" file-path-separator "worker.xml") + (str "-Dlogging.sensitivity=" mock-sensitivity) + (str "-Dlog4j.configurationFile=" file-path-separator "log4j2" file-path-separator "worker.xml") (str "-Dstorm.id=" mock-storm-id) (str "-Dworker.id=" mock-worker-id) (str "-Dworker.port=" mock-port) @@ -364,6 +375,7 @@ (let [mock-port "42" mock-storm-id "fake-storm-id" mock-worker-id "fake-worker-id" + mock-sensitivity "S3" mock-cp "mock-classpath'quote-on-purpose" storm-local (str "/tmp/" (UUID/randomUUID)) worker-script (str storm-local "/workers/" mock-worker-id "/storm-worker-script.sh") @@ -373,7 +385,16 @@ (str storm-local "/workers/" mock-worker-id) worker-script] exp-script-fn (fn [opts topo-opts] - (str "#!/bin/bash\n'export' 'LD_LIBRARY_PATH=';\n\nexec 'java' '-server'" + (str "#!/bin/bash\n'export' 'LD_LIBRARY_PATH=';\n\nexec 'java'" + " '-cp' 'mock-classpath'\"'\"'quote-on-purpose'" + " '-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log'" + " '-Dstorm.home='" + " '-Dstorm.id=" mock-storm-id "'" + " '-Dworker.id=" mock-worker-id "'" + " '-Dworker.port=" mock-port "'" + " '-Dlog4j.configurationFile=/log4j2/worker.xml'" + " 'backtype.storm.LogWriter'" + " 'java' '-server'" " " (shell-cmd opts) " " (shell-cmd topo-opts) " '-Djava.library.path='" @@ -382,7 +403,8 @@ " '-Dstorm.conf.file='" " '-Dstorm.options='" " '-Dstorm.log.dir=/logs'" - " '-Dlogback.configurationFile=/logback/worker.xml'" + " '-Dlogging.sensitivity=" mock-sensitivity "'" + " '-Dlog4j.configurationFile=/log4j2/worker.xml'" " '-Dstorm.id=" mock-storm-id "'" " '-Dworker.id=" mock-worker-id "'" " '-Dworker.port=" mock-port "'" http://git-wip-us.apache.org/repos/asf/storm/blob/ce60f3b5/storm-dist/binary/src/main/assembly/binary.xml ---------------------------------------------------------------------- diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml index 1328a8c..d35ab96 100644 --- a/storm-dist/binary/src/main/assembly/binary.xml +++ b/storm-dist/binary/src/main/assembly/binary.xml @@ -260,13 +260,13 @@ </file> <!-- TODO this should be a generated file from "target" --> <file> - <source>${project.basedir}/../../logback/cluster.xml</source> - <outputDirectory>/logback</outputDirectory> + <source>${project.basedir}/../../log4j2/cluster.xml</source> + <outputDirectory>/log4j2</outputDirectory> </file> <file> - <source>${project.basedir}/../../logback/worker.xml</source> - <outputDirectory>/logback</outputDirectory> + <source>${project.basedir}/../../log4j2/worker.xml</source> + <outputDirectory>/log4j2</outputDirectory> </file> <file>
