Repository: logging-log4j2 Updated Branches: refs/heads/master 6e4ff7349 -> d42e20584
LOG4J2-649 - Add PurgePolicy and IdlePurgePolicy to RoutingAppender. Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo Commit: http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/d42e2058 Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/d42e2058 Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/d42e2058 Branch: refs/heads/master Commit: d42e205844da3fffb224e55b31fc2fe3ac14ca93 Parents: 6e4ff73 Author: Ralph Goers <[email protected]> Authored: Thu Nov 26 22:59:36 2015 -0700 Committer: Ralph Goers <[email protected]> Committed: Thu Nov 26 22:59:36 2015 -0700 ---------------------------------------------------------------------- .../core/appender/routing/IdlePurgePolicy.java | 141 +++++++++++++++++++ .../core/appender/routing/PurgePolicy.java | 31 ++++ .../core/appender/routing/RoutingAppender.java | 32 ++++- .../core/config/ConfigurationScheduler.java | 4 + .../routing/RoutingAppenderWithPurgingTest.java | 108 ++++++++++++++ .../src/test/resources/log4j-routing-purge.xml | 71 ++++++++++ src/changes/changes.xml | 3 + src/site/xdoc/manual/appenders.xml | 8 ++ 8 files changed, 395 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/d42e2058/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java new file mode 100644 index 0000000..b9a0a26 --- /dev/null +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/IdlePurgePolicy.java @@ -0,0 +1,141 @@ +package org.apache.logging.log4j.core.appender.routing; + +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.AbstractLifeCycle; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.ConfigurationScheduler; +import org.apache.logging.log4j.core.config.Scheduled; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginConfiguration; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.status.StatusLogger; + +/** + * + * Policy is purging appenders that were not in use specified time in minutes + * + */ +@Plugin(name = "IdlePurgePolicy", category = "Core", printObject = true) +@Scheduled +public class IdlePurgePolicy extends AbstractLifeCycle implements PurgePolicy, Runnable { + + private static final Logger LOGGER = StatusLogger.getLogger(); + private final long timeToLive; + private final ConcurrentMap<String, Long> appendersUsage = new ConcurrentHashMap<>(); + private RoutingAppender routingAppender; + private final ConfigurationScheduler scheduler; + private volatile ScheduledFuture<?> future = null; + + public IdlePurgePolicy(long timeToLive, ConfigurationScheduler scheduler) { + this.timeToLive = timeToLive; + this.scheduler = scheduler; + } + + @Override + public void initialize(RoutingAppender routingAppender) { + this.routingAppender = routingAppender; + } + + @Override + public void stop() { + super.stop(); + future.cancel(true); + } + + /** + * Purging appenders that were not in use specified time + * + */ + @Override + public void purge() { + long createTime = System.currentTimeMillis() - timeToLive; + for (Entry<String, Long> entry : appendersUsage.entrySet()) { + if (entry.getValue() < createTime) { + LOGGER.debug("Removing appender " + entry.getKey()); + appendersUsage.remove(entry.getKey()); + routingAppender.deleteAppender(entry.getKey()); + } + } + } + + @Override + public void update(String key, LogEvent event) { + long now = System.currentTimeMillis(); + appendersUsage.put(key, now); + if (future == null) { + synchronized(this) { + if (future == null) { + scheduleNext(); + } + } + } + + } + + @Override + public void run() { + purge(); + scheduleNext(); + } + + private void scheduleNext() { + long createTime = Long.MAX_VALUE; + for (Entry<String, Long> entry : appendersUsage.entrySet()) { + if (entry.getValue() < createTime) { + createTime = entry.getValue(); + } + } + if (createTime < Long.MAX_VALUE) { + long interval = timeToLive - (System.currentTimeMillis() - createTime); + future = scheduler.schedule(this, interval, TimeUnit.MILLISECONDS); + } + } + + /** + * Create the PurgePolicy + * @param timeToLive the number of increments of timeUnit before the Appender should be purged. + * @param timeUnit the unit of time the timeToLive is expressed in. + * @return The Routes container. + */ + @PluginFactory + public static PurgePolicy createPurgePolicy( + @PluginAttribute("timeToLive") final String timeToLive, + @PluginAttribute("timeUnit") final String timeUnit, + @PluginConfiguration Configuration configuration) { + + if (timeToLive == null) { + LOGGER.error("A timeToLive value is required"); + return null; + } + TimeUnit units; + if (timeUnit == null) { + units = TimeUnit.MINUTES; + } else { + try { + units = TimeUnit.valueOf(timeUnit.toUpperCase()); + } catch(Exception ex) { + LOGGER.error("Invalid time unit {}", timeUnit); + units = TimeUnit.MINUTES; + } + } + + final long ttl = units.toMillis(Long.parseLong(timeToLive)); + + + return new IdlePurgePolicy(ttl, configuration.getScheduler()); + } + + @Override + public String toString() { + return "timeToLive=" + timeToLive; + } + +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/d42e2058/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java new file mode 100644 index 0000000..cfac2fa --- /dev/null +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/PurgePolicy.java @@ -0,0 +1,31 @@ +package org.apache.logging.log4j.core.appender.routing; + +import org.apache.logging.log4j.core.LogEvent; + +/** + * + * Policy for purging routed appenders + * + */ +public interface PurgePolicy { + + /** + * Activate purging appenders + */ + void purge(); + + /** + * + * @param routed appender key + * @param event + */ + void update(String key, LogEvent event); + + /** + * Initialize with routing appender + * + * @param routingAppender + */ + void initialize(RoutingAppender routingAppender); + +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/d42e2058/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java index 02be2c2..01f63b7 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/routing/RoutingAppender.java @@ -16,6 +16,7 @@ */ package org.apache.logging.log4j.core.appender.routing; +import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -52,13 +53,18 @@ public final class RoutingAppender extends AbstractAppender { private final Configuration config; private final ConcurrentMap<String, AppenderControl> appenders = new ConcurrentHashMap<>(); private final RewritePolicy rewritePolicy; + private final PurgePolicy purgePolicy; private RoutingAppender(final String name, final Filter filter, final boolean ignoreExceptions, final Routes routes, - final RewritePolicy rewritePolicy, final Configuration config) { + final RewritePolicy rewritePolicy, final Configuration config, PurgePolicy purgePolicy) { super(name, filter, null, ignoreExceptions); this.routes = routes; this.config = config; this.rewritePolicy = rewritePolicy; + this.purgePolicy = purgePolicy; + if(this.purgePolicy != null) { + this.purgePolicy.initialize(this); + } Route defRoute = null; for (final Route route : routes.getRoutes()) { if (route.getKey() == null) { @@ -111,9 +117,13 @@ public final class RoutingAppender extends AbstractAppender { if (control != null) { control.callAppender(event); } + + if(purgePolicy != null) { + purgePolicy.update(key, event); + } } - private synchronized AppenderControl getControl(final String key, final LogEvent event) { + private synchronized AppenderControl getControl(final String key, final LogEvent event) { AppenderControl control = appenders.get(key); if (control != null) { return control; @@ -162,6 +172,21 @@ public final class RoutingAppender extends AbstractAppender { LOGGER.error("No Appender was configured for route " + route.getKey()); return null; } + + public Map<String, AppenderControl> getAppenders() { + return Collections.unmodifiableMap(appenders); + } + + /** + * Delete specified appender + * + * @param key The appender's key + */ + public void deleteAppender(String key) { + LOGGER.debug("Stopping route with key" + key); + AppenderControl control = appenders.remove(key); + control.getAppender().stop(); + } /** * Create a RoutingAppender. @@ -181,6 +206,7 @@ public final class RoutingAppender extends AbstractAppender { @PluginElement("Routes") final Routes routes, @PluginConfiguration final Configuration config, @PluginElement("RewritePolicy") final RewritePolicy rewritePolicy, + @PluginElement("PurgePolicy") final PurgePolicy purgePolicy, @PluginElement("Filter") final Filter filter) { final boolean ignoreExceptions = Booleans.parseBoolean(ignore, true); @@ -192,6 +218,6 @@ public final class RoutingAppender extends AbstractAppender { LOGGER.error("No routes defined for RoutingAppender"); return null; } - return new RoutingAppender(name, filter, ignoreExceptions, routes, rewritePolicy, config); + return new RoutingAppender(name, filter, ignoreExceptions, routes, rewritePolicy, config, purgePolicy); } } http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/d42e2058/log4j-core/src/main/java/org/apache/logging/log4j/core/config/ConfigurationScheduler.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/ConfigurationScheduler.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/ConfigurationScheduler.java index fcd81d9..33e3225 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/ConfigurationScheduler.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/ConfigurationScheduler.java @@ -49,6 +49,8 @@ public class ConfigurationScheduler extends AbstractLifeCycle { scheduledItems = 5; } executorService = new ScheduledThreadPoolExecutor(scheduledItems, new DaemonThreadFactory("Log4j2Scheduled-")); + } else { + LOGGER.debug("No scheduled items"); } } @@ -67,6 +69,8 @@ public class ConfigurationScheduler extends AbstractLifeCycle { public void incrementScheduledItems() { if (!isStarted()) { ++scheduledItems; + } else { + LOGGER.error("Attempted to increment scheduled items after start"); } } http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/d42e2058/log4j-core/src/test/java/org/apache/logging/log4j/core/appender/routing/RoutingAppenderWithPurgingTest.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/appender/routing/RoutingAppenderWithPurgingTest.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/appender/routing/RoutingAppenderWithPurgingTest.java new file mode 100644 index 0000000..1c3df3d --- /dev/null +++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/appender/routing/RoutingAppenderWithPurgingTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.apache.logging.log4j.core.appender.routing; + +import static org.junit.Assert.*; + +import java.io.File; +import java.util.List; + +import org.apache.logging.log4j.EventLogger; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.junit.CleanFiles; +import org.apache.logging.log4j.junit.LoggerContextRule; +import org.apache.logging.log4j.message.StructuredDataMessage; +import org.apache.logging.log4j.test.appender.ListAppender; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +/** + * Testing Routing appender purge facilities + */ +public class RoutingAppenderWithPurgingTest { + private static final String CONFIG = "log4j-routing-purge.xml"; + private static final String IDLE_LOG_FILE1 = "target/routing-purge-idle/routingtest-1.log"; + private static final String IDLE_LOG_FILE2 = "target/routing-purge-idle/routingtest-2.log"; + private static final String IDLE_LOG_FILE3 = "target/routing-purge-idle/routingtest-3.log"; + private static final String MANUAL_LOG_FILE1 = "target/routing-purge-manual/routingtest-1.log"; + private static final String MANUAL_LOG_FILE2 = "target/routing-purge-manual/routingtest-2.log"; + private static final String MANUAL_LOG_FILE3 = "target/routing-purge-manual/routingtest-3.log"; + + + private ListAppender app; + private RoutingAppender routingAppenderIdle; + private RoutingAppender routingAppenderManual; + + @Rule + public LoggerContextRule init = new LoggerContextRule(CONFIG); + + @Rule + public CleanFiles files = new CleanFiles(IDLE_LOG_FILE1, IDLE_LOG_FILE2, IDLE_LOG_FILE3, + MANUAL_LOG_FILE1, MANUAL_LOG_FILE2, MANUAL_LOG_FILE3); + + + @Before + public void setUp() throws Exception { + this.app = this.init.getListAppender("List"); + this.routingAppenderIdle = this.init.getRequiredAppender("RoutingPurgeIdle", RoutingAppender.class); + this.routingAppenderManual = this.init.getRequiredAppender("RoutingPurgeManual", RoutingAppender.class); + } + + @After + public void tearDown() throws Exception { + this.app.clear(); + } + + @Test + public void routingTest() throws InterruptedException { + StructuredDataMessage msg = new StructuredDataMessage("1", "This is a test 1", "Service"); + EventLogger.logEvent(msg); + final List<LogEvent> list = app.getEvents(); + assertNotNull("No events generated", list); + assertTrue("Incorrect number of events. Expected 1, got " + list.size(), list.size() == 1); + msg = new StructuredDataMessage("2", "This is a test 2", "Service"); + EventLogger.logEvent(msg); + msg = new StructuredDataMessage("3", "This is a test 3", "Service"); + EventLogger.logEvent(msg); + String[] files = {IDLE_LOG_FILE1, IDLE_LOG_FILE2, IDLE_LOG_FILE3, MANUAL_LOG_FILE1, MANUAL_LOG_FILE2, MANUAL_LOG_FILE3}; + assertFileExistance(files); + + assertEquals("Incorrect number of appenders with IdlePurgePolicy.", 3, routingAppenderIdle.getAppenders().size()); + assertEquals("Incorrect number of appenders manual purge.", 3, routingAppenderManual.getAppenders().size()); + + Thread.sleep(3000); + EventLogger.logEvent(msg); + + assertEquals("Incorrect number of appenders with IdlePurgePolicy.", 1, routingAppenderIdle.getAppenders().size()); + assertEquals("Incorrect number of appenders with manual purge.", 3, routingAppenderManual.getAppenders().size()); + + routingAppenderManual.deleteAppender("1"); + routingAppenderManual.deleteAppender("2"); + routingAppenderManual.deleteAppender("3"); + + assertEquals("Incorrect number of appenders with IdlePurgePolicy.", 1, routingAppenderIdle.getAppenders().size()); + assertEquals("Incorrect number of appenders with manual purge.", 0, routingAppenderManual.getAppenders().size()); + } + + private void assertFileExistance(String... files) { + for (String file : files) { + assertTrue("File should exist - " + file + " file ", new File(file).exists()); + } + } +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/d42e2058/log4j-core/src/test/resources/log4j-routing-purge.xml ---------------------------------------------------------------------- diff --git a/log4j-core/src/test/resources/log4j-routing-purge.xml b/log4j-core/src/test/resources/log4j-routing-purge.xml new file mode 100644 index 0000000..995148b --- /dev/null +++ b/log4j-core/src/test/resources/log4j-routing-purge.xml @@ -0,0 +1,71 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<Configuration status="OFF" name="RoutingTest"> + <Properties> + <Property name="filename-idle">target/routing-purge-idle/routingtest-$${sd:id}.log</Property> + <Property name="filename-manual">target/routing-purge-manual/routingtest-$${sd:id}.log</Property> + </Properties> + <ThresholdFilter level="debug"/> + + <Appenders> + <Console name="STDOUT"> + <PatternLayout pattern="%m%n"/> + </Console> + <List name="List"> + <ThresholdFilter level="debug"/> + </List> + <Routing name="RoutingPurgeIdle"> + <Routes pattern="$${sd:id}"> + <Route> + <File name="Routing-${sd:id}" fileName="${filename-idle}"> + <PatternLayout> + <Pattern>%d %p %C{1.} [%t] %m%n</Pattern> + </PatternLayout> + </File> + </Route> + </Routes> + <IdlePurgePolicy timeToLive="2" timeUnit="seconds" /> + </Routing> + + <Routing name="RoutingPurgeManual"> + <Routes pattern="$${sd:id}"> + <Route> + <File name="Routing-${sd:id}" fileName="${filename-manual}"> + <PatternLayout> + <Pattern>%d %p %C{1.} [%t] %m%n</Pattern> + </PatternLayout> + </File> + </Route> + </Routes> + </Routing> + </Appenders> + + <Loggers> + <Logger name="EventLogger" level="info" additivity="false"> + <AppenderRef ref="RoutingPurgeIdle"/> + <AppenderRef ref="RoutingPurgeManual"/> + <AppenderRef ref="List"/> + </Logger> + + <Root level="error"> + <AppenderRef ref="STDOUT"/> + </Root> + </Loggers> + +</Configuration> http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/d42e2058/src/changes/changes.xml ---------------------------------------------------------------------- diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 71059cd..ab94586 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -24,6 +24,9 @@ </properties> <body> <release version="2.5" date="2015-MM-DD" description="GA Release 2.5"> + <action issue="LOG4J2-649" dev="rgoers" type="update" due-to="Aleksey Zvolinsky"> + Add PurgePolicy and IdlePurgePolicy to RoutingAppender. + </action> <action issue="LOG4J2-1202" dev="rgoers" type="update"> Remove ConfigurationMonitor. The WatchManager is now used to check for configuration changes. </action> http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/d42e2058/src/site/xdoc/manual/appenders.xml ---------------------------------------------------------------------- diff --git a/src/site/xdoc/manual/appenders.xml b/src/site/xdoc/manual/appenders.xml index 549fed4..1f16be6 100644 --- a/src/site/xdoc/manual/appenders.xml +++ b/src/site/xdoc/manual/appenders.xml @@ -2795,6 +2795,13 @@ public class JpaLogEntity extends AbstractLogEventWrapperEntity { Appender definition then an Appender will be created within the context of the RoutingAppender and will be reused each time a matching Appender name is referenced through a Route. </p> + <h4>Purge Policy</h4> + <p>The RoutingAppender can be configured with a PurgePolicy whose purpose is to stop and remove dormant + Appenders that have been dynamically created by the RoutingAppender. Log4j currently provides the + IdlePurgePolicy as the only PurgePolicy available for cleaning up the Appenders. The IdlePurgePolicy + accepts 2 attributes; timeToLive, which is the number of timeUnits the Appender should survive without + having any events sent to it, and timeUnit, the String representation of java.util.concurrent.TimeUnit + which is used with the timeToLive attribute.</p> <p> Below is a sample configuration that uses a RoutingAppender to route all Audit events to a FlumeAppender and all other events will be routed to a RollingFileAppender that captures only @@ -2823,6 +2830,7 @@ public class JpaLogEntity extends AbstractLogEventWrapperEntity { </Route> <Route ref="AuditLogger" key="Audit"/> </Routes> + <IdlePurgePolicy timeToLive="15" timeUnit="minutes"/> </Routing> </Appenders> <Loggers>
