Repository: storm Updated Branches: refs/heads/master 0e80f81f4 -> f75fddec9
create access logging for all daemons removed a file Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/064ff8bb Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/064ff8bb Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/064ff8bb Branch: refs/heads/master Commit: 064ff8bb3e9b037017f87ab2576dbbb091de9c8a Parents: 1d393ee Author: Sanket <schintap@untilservice-lm> Authored: Sat Oct 17 23:24:05 2015 -0500 Committer: Sanket <schintap@untilservice-lm> Committed: Sun Oct 18 10:12:18 2015 -0500 ---------------------------------------------------------------------- log4j2/cluster.xml | 34 +++++++++---- .../src/clj/backtype/storm/daemon/drpc.clj | 2 + .../src/clj/backtype/storm/daemon/nimbus.clj | 3 +- .../src/clj/backtype/storm/ui/helpers.clj | 5 ++ storm-core/src/clj/backtype/storm/util.clj | 5 ++ .../storm/logging/ThriftAccessLogger.java | 27 ++++++++++ .../logging/filters/AccessLoggingFilter.java | 52 ++++++++++++++++++++ .../auth/authorizer/DenyAuthorizer.java | 5 -- .../auth/authorizer/NoopAuthorizer.java | 5 -- .../auth/authorizer/SimpleACLAuthorizer.java | 6 --- .../authorizer/SimpleWhitelistAuthorizer.java | 5 -- 11 files changed, 117 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/log4j2/cluster.xml ---------------------------------------------------------------------- diff --git a/log4j2/cluster.xml b/log4j2/cluster.xml index bb96646..1ced014 100644 --- a/log4j2/cluster.xml +++ b/log4j2/cluster.xml @@ -19,12 +19,12 @@ <configuration monitorInterval="60"> <properties> <property name="pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} [%p] %msg%n</property> - <property name="patternMetris">%d %-8r %m%n</property> + <property name="patternMetrics">%d %-8r %m%n</property> </properties> <appenders> <RollingFile name="A1" fileName="${sys:storm.log.dir}/${sys:logfile.name}" - filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i"> + filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i.gz"> <PatternLayout> <pattern>${pattern}</pattern> </PatternLayout> @@ -33,9 +33,9 @@ </Policies> <DefaultRolloverStrategy max="9"/> </RollingFile> - <RollingFile name="ACCESS" - fileName="${sys:storm.log.dir}/access.log" - filePattern="${sys:storm.log.dir}/access.log.%i"> + <RollingFile name="WEB-ACCESS" + fileName="${sys:storm.log.dir}/access-web-${sys:daemon.name}.log" + filePattern="${sys:storm.log.dir}/access-web-${sys:daemon.name}.log.%i.gz"> <PatternLayout> <pattern>${pattern}</pattern> </PatternLayout> @@ -44,11 +44,22 @@ </Policies> <DefaultRolloverStrategy max="9"/> </RollingFile> + <RollingFile name="THRIFT-ACCESS" + fileName="${sys:storm.log.dir}/access-${sys:logfile.name}" + filePattern="${sys:storm.log.dir}/access-${sys:logfile.name}.%i.gz"> + <PatternLayout> + <pattern>${pattern}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + </Policies> + <DefaultRolloverStrategy max="9"/> + </RollingFile> <RollingFile name="METRICS" fileName="${sys:storm.log.dir}/metrics.log" - filePattern="${sys:storm.log.dir}/metrics.log.%i"> + filePattern="${sys:storm.log.dir}/metrics.log.%i.gz"> <PatternLayout> - <pattern>${patternMetris}</pattern> + <pattern>${patternMetrics}</pattern> </PatternLayout> <Policies> <SizeBasedTriggeringPolicy size="2 MB"/> <!-- Or every 100 MB --> @@ -62,8 +73,13 @@ </appenders> <loggers> - <Logger name="backtype.storm.security.auth.authorizer" level="info"> - <AppenderRef ref="ACCESS"/> + <Logger name="backtype.storm.logging.filters.AccessLoggingFilter" level="info" additivity="false"> + <AppenderRef ref="WEB-ACCESS"/> + <AppenderRef ref="syslog"/> + </Logger> + <Logger name="backtype.storm.logging.ThriftAccessLogger" level="info" additivity="false"> + <AppenderRef ref="THRIFT-ACCESS"/> + <AppenderRef ref="syslog"/> </Logger> <Logger name="backtype.storm.metric.LoggingMetricsConsumer" level="info"> <AppenderRef ref="METRICS"/> http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/storm-core/src/clj/backtype/storm/daemon/drpc.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/daemon/drpc.clj b/storm-core/src/clj/backtype/storm/daemon/drpc.clj index 9475910..abaf2e7 100644 --- a/storm-core/src/clj/backtype/storm/daemon/drpc.clj +++ b/storm-core/src/clj/backtype/storm/daemon/drpc.clj @@ -45,6 +45,8 @@ (defn check-authorization ([aclHandler mapping operation context] + (if (not-nil? context) + (log-thrift-access (.requestID context) (.remoteAddress context) (.principal context) operation)) (if aclHandler (let [context (or context (ReqContext/context))] (if-not (.permit aclHandler context operation mapping) http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/storm-core/src/clj/backtype/storm/daemon/nimbus.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj index eee7bac..1d711fa 100644 --- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj +++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj @@ -831,8 +831,7 @@ impersonation-authorizer (:impersonation-authorization-handler nimbus) ctx (or context (ReqContext/context)) check-conf (if storm-conf storm-conf (if storm-name {TOPOLOGY-NAME storm-name}))] - (log-message "[req " (.requestID ctx) "] Access from: " (.remoteAddress ctx) " principal:" (.principal ctx) " op:" operation) - + (log-thrift-access (.requestID ctx) (.remoteAddress ctx) (.principal ctx) operation) (if (.isImpersonating ctx) (do (log-warn "principal: " (.realPrincipal ctx) " is trying to impersonate principal: " (.principal ctx)) http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/storm-core/src/clj/backtype/storm/ui/helpers.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/ui/helpers.clj b/storm-core/src/clj/backtype/storm/ui/helpers.clj index 3be33b4..cbbff85 100644 --- a/storm-core/src/clj/backtype/storm/ui/helpers.clj +++ b/storm-core/src/clj/backtype/storm/ui/helpers.clj @@ -23,6 +23,7 @@ (:use [backtype.storm.util :only [clojurify-structure uuid defnk url-encode not-nil?]]) (:use [clj-time coerce format]) (:import [backtype.storm.generated ExecutorInfo ExecutorSummary]) + (:import [backtype.storm.logging.filters AccessLoggingFilter]) (:import [java.util EnumSet]) (:import [org.eclipse.jetty.server Server] [org.eclipse.jetty.server.nio SelectChannelConnector] @@ -143,6 +144,9 @@ (.setInitParameter CrossOriginFilter/ACCESS_CONTROL_ALLOW_ORIGIN_HEADER "*") )) +(defn mk-access-logging-filter-handler [] + (org.eclipse.jetty.servlet.FilterHolder. (AccessLoggingFilter.))) + (defn config-filter [server handler filters-confs] (if filters-confs (let [servlet-holder (ServletHolder. @@ -157,6 +161,7 @@ (.setName (or filter-name filter-class)) (.setInitParameters (or filter-params {})))] (.addFilter context filter-holder "/*" FilterMapping/ALL)))) + (.addFilter context (mk-access-logging-filter-handler) "/*" (EnumSet/allOf DispatcherType)) (.setHandler server context)))) (defn ring-response-from-exception [ex] http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/storm-core/src/clj/backtype/storm/util.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj index 983f282..19e52a6 100644 --- a/storm-core/src/clj/backtype/storm/util.clj +++ b/storm-core/src/clj/backtype/storm/util.clj @@ -31,6 +31,7 @@ (:import [java.lang.management ManagementFactory]) (:import [org.apache.commons.exec DefaultExecutor CommandLine]) (:import [org.apache.commons.io FileUtils]) + (:import [backtype.storm.logging ThriftAccessLogger]) (:import [org.apache.commons.exec ExecuteException]) (:import [org.json.simple JSONValue]) (:import [org.yaml.snakeyaml Yaml] @@ -1070,3 +1071,7 @@ (assoc coll k (apply str (repeat (count (coll k)) "#"))) coll)) +(defn log-thrift-access [request-id remoteAddress principal operation] + (doto + (ThriftAccessLogger.) + (.log (str "Request ID: " request-id " access from: " remoteAddress " principal: " principal " operation: " operation)))) http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/storm-core/src/jvm/backtype/storm/logging/ThriftAccessLogger.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/logging/ThriftAccessLogger.java b/storm-core/src/jvm/backtype/storm/logging/ThriftAccessLogger.java new file mode 100644 index 0000000..3374af0 --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/logging/ThriftAccessLogger.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package backtype.storm.logging; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ThriftAccessLogger { + private static final Logger LOG = LoggerFactory.getLogger(ThriftAccessLogger.class); + public void log(String logMessage) { + LOG.info(logMessage); + } +} http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/storm-core/src/jvm/backtype/storm/logging/filters/AccessLoggingFilter.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/logging/filters/AccessLoggingFilter.java b/storm-core/src/jvm/backtype/storm/logging/filters/AccessLoggingFilter.java new file mode 100644 index 0000000..6c368f1 --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/logging/filters/AccessLoggingFilter.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package backtype.storm.logging.filters; +import java.io.IOException; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AccessLoggingFilter implements Filter { + + private static final Logger LOG = LoggerFactory.getLogger(AccessLoggingFilter.class); + public void init(FilterConfig config) throws ServletException { + } + + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + handle((HttpServletRequest)request, (HttpServletResponse)response, chain); + } + + public void handle(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException{ + if (request != null) { + LOG.info("Access from: {} url: {} principal: {}", request.getRemoteAddr(), request.getRequestURL(), + (request.getUserPrincipal() == null ? "" : request.getUserPrincipal().getName())); + } + chain.doFilter(request, response); + } + + public void destroy() { + } +} http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/storm-core/src/jvm/backtype/storm/security/auth/authorizer/DenyAuthorizer.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/DenyAuthorizer.java b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/DenyAuthorizer.java index 5e84b38..8d61492 100644 --- a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/DenyAuthorizer.java +++ b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/DenyAuthorizer.java @@ -47,11 +47,6 @@ public class DenyAuthorizer implements IAuthorizer { * @return true if the request is authorized, false if reject */ public boolean permit(ReqContext context, String operation, Map topology_conf) { - LOG.info("[req "+ context.requestID()+ "] Access " - + " from: " + (context.remoteAddress() == null? "null" : context.remoteAddress().toString()) - + (context.principal() == null? "" : (" principal:"+ context.principal())) - +" op:"+operation - + (topology_conf == null? "" : (" topoology:"+topology_conf.get(Config.TOPOLOGY_NAME)))); return false; } } http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/storm-core/src/jvm/backtype/storm/security/auth/authorizer/NoopAuthorizer.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/NoopAuthorizer.java b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/NoopAuthorizer.java index 9af44d3..c8008f1 100644 --- a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/NoopAuthorizer.java +++ b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/NoopAuthorizer.java @@ -47,11 +47,6 @@ public class NoopAuthorizer implements IAuthorizer { * @return true if the request is authorized, false if reject */ public boolean permit(ReqContext context, String operation, Map topology_conf) { - LOG.info("[req "+ context.requestID()+ "] Access " - + " from: " + (context.remoteAddress() == null? "null" : context.remoteAddress().toString()) - + (context.principal() == null? "" : (" principal:"+ context.principal())) - +" op:"+operation - + (topology_conf == null? "" : (" topoology:"+topology_conf.get(Config.TOPOLOGY_NAME)))); return true; } } http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java index c77d201..e856578 100644 --- a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java +++ b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleACLAuthorizer.java @@ -103,12 +103,6 @@ public class SimpleACLAuthorizer implements IAuthorizer { */ @Override public boolean permit(ReqContext context, String operation, Map topology_conf) { - LOG.info("[req " + context.requestID() + "] Access " - + " from: " + (context.remoteAddress() == null ? "null" : context.remoteAddress().toString()) - + (context.principal() == null ? "" : (" principal:" + context.principal())) - + " op:" + operation - + (topology_conf == null ? "" : (" topoology:" + topology_conf.get(Config.TOPOLOGY_NAME)))); - String principal = context.principal().getName(); String user = _ptol.toLocal(context.principal()); Set<String> userGroups = new HashSet<String>(); http://git-wip-us.apache.org/repos/asf/storm/blob/064ff8bb/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleWhitelistAuthorizer.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleWhitelistAuthorizer.java b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleWhitelistAuthorizer.java index 55109f9..c522adf 100644 --- a/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleWhitelistAuthorizer.java +++ b/storm-core/src/jvm/backtype/storm/security/auth/authorizer/SimpleWhitelistAuthorizer.java @@ -60,11 +60,6 @@ public class SimpleWhitelistAuthorizer implements IAuthorizer { */ @Override public boolean permit(ReqContext context, String operation, Map topology_conf) { - LOG.info("[req "+ context.requestID()+ "] Access " - + " from: " + (context.remoteAddress() == null? "null" : context.remoteAddress().toString()) - + (context.principal() == null? "" : (" principal:"+ context.principal())) - +" op:"+operation - + (topology_conf == null? "" : (" topoology:"+topology_conf.get(Config.TOPOLOGY_NAME)))); return context.principal() != null ? users.contains(context.principal().getName()) : false; } }
