This is an automated email from the ASF dual-hosted git repository. rgoers pushed a commit to branch release-2.x in repository https://gitbox.apache.org/repos/asf/logging-log4j2.git
commit e5f4479e25ce76a66b7c4af78bbbd626e2f34ac5 Author: Ralph Goers <[email protected]> AuthorDate: Sun Mar 31 23:05:33 2019 -0700 LOG4J2-913 - Add more tests and update site --- .../core/appender/mom/kafka/KafkaAppender.java | 8 +- .../core/appender/mom/kafka/KafkaManager.java | 39 ++++ .../log4j/core/layout/AbstractJacksonLayout.java | 142 +++++++++++++- log4j-docker/pom.xml | 21 ++ .../logging/log4j/docker/model/Container.java | 6 + .../logging/log4j/docker/model/IPAMConfig.java | 11 ++ .../apache/logging/log4j/docker/model/Network.java | 12 +- .../log4j/docker/model/NetworkSettings.java | 177 +++++++++++++++++ log4j-docker/src/site/markdown/index.md.vm | 55 ++++++ log4j-docker/src/site/site.xml | 52 +++++ .../logging/log4j/flume/appender/BatchEvent.java | 4 + .../log4j/flume/appender/FlumeAppender.java | 1 - .../log4j/flume/appender/FlumeAvroManager.java | 42 ++-- log4j-samples/log4j-samples-flume-embedded/pom.xml | 1 + log4j-samples/log4j-samples-flume-remote/pom.xml | 1 + .../log4j-spring-cloud-config-client/pom.xml | 4 +- .../src/site/markdown/index.md | 88 +++++++++ .../src/site/site.xml | 52 +++++ .../README.md | 74 +++---- .../docker/app-compose.yml | 21 ++ .../docker/docker-compose.yml | 1 + .../pom.xml | 16 ++ .../pom.xml | 20 +- .../src/main/config-repo/log4j2.xml | 53 +++++- .../src/main/resources/log4j2.xml | 2 +- log4j-spring-cloud-config/pom.xml | 1 + .../src/site/markdown/index.md | 21 ++ log4j-spring-cloud-config/src/site/site.xml | 52 +++++ src/site/markdown/index.md.vm | 6 + src/site/markdown/manual/cloud.md | 212 +++++++++++++++++++++ src/site/resources/images/DockerFluentd.drawio | 1 + src/site/resources/images/DockerFluentd.png | Bin 0 -> 16444 bytes .../images/DockerFluentdAggregator.drawio | 1 + .../resources/images/DockerFluentdAggregator.png | Bin 0 -> 15394 bytes src/site/resources/images/DockerLogFile.drawio | 1 + src/site/resources/images/DockerLogFile.png | Bin 0 -> 14674 bytes src/site/resources/images/DockerStdout.drawio | 1 + src/site/resources/images/DockerStdout.png | Bin 0 -> 19551 bytes src/site/resources/images/DockerTCP.drawio | 1 + src/site/resources/images/DockerTCP.png | Bin 0 -> 11895 bytes src/site/resources/images/LoggerAggregator.drawio | 1 + src/site/resources/images/LoggerAggregator.png | Bin 0 -> 11342 bytes src/site/site.xml | 4 + src/site/xdoc/manual/configuration.xml.vm | 78 ++++++++ src/site/xdoc/manual/lookups.xml | 25 +++ src/site/xdoc/runtime-dependencies.xml | 74 ++++--- 46 files changed, 1265 insertions(+), 117 deletions(-) diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/mom/kafka/KafkaAppender.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/mom/kafka/KafkaAppender.java index ef4d656..73d95a8 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/mom/kafka/KafkaAppender.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/mom/kafka/KafkaAppender.java @@ -67,8 +67,8 @@ public final class KafkaAppender extends AbstractAppender { AbstractLifeCycle.LOGGER.error("No layout provided for KafkaAppender"); return null; } - final KafkaManager kafkaManager = new KafkaManager(getConfiguration().getLoggerContext(), getName(), topic, - syncSend, getPropertyArray(), key); + final KafkaManager kafkaManager = KafkaManager.getManager(getConfiguration().getLoggerContext(), + getName(), topic, syncSend, getPropertyArray(), key); return new KafkaAppender(getName(), layout, getFilter(), isIgnoreExceptions(), kafkaManager, getPropertyArray()); } @@ -108,8 +108,8 @@ public final class KafkaAppender extends AbstractAppender { AbstractLifeCycle.LOGGER.error("No layout provided for KafkaAppender"); return null; } - final KafkaManager kafkaManager = - new KafkaManager(configuration.getLoggerContext(), name, topic, true, properties, key); + final KafkaManager kafkaManager = KafkaManager.getManager(configuration.getLoggerContext(), name, topic, + true, properties, key); return new KafkaAppender(name, layout, filter, ignoreExceptions, kafkaManager, null); } diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/mom/kafka/KafkaManager.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/mom/kafka/KafkaManager.java index 9af11cd..8db4204 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/mom/kafka/KafkaManager.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/mom/kafka/KafkaManager.java @@ -31,6 +31,7 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.AbstractManager; +import org.apache.logging.log4j.core.appender.ManagerFactory; import org.apache.logging.log4j.core.config.Property; import org.apache.logging.log4j.core.util.Log4jThread; @@ -50,7 +51,11 @@ public class KafkaManager extends AbstractManager { private final String topic; private final String key; private final boolean syncSend; + private static final KafkaManagerFactory factory = new KafkaManagerFactory(); + /* + * The Constructor should have been declared private as all Managers are create by the internal factory; + */ public KafkaManager(final LoggerContext loggerContext, final String name, final String topic, final boolean syncSend, final Property[] properties, final String key) { super(loggerContext, name); @@ -135,4 +140,38 @@ public class KafkaManager extends AbstractManager { return topic; } + public static KafkaManager getManager(final LoggerContext loggerContext, final String name, final String topic, + final boolean syncSend, final Property[] properties, final String key) { + StringBuilder sb = new StringBuilder(name); + for (Property prop: properties) { + sb.append(" ").append(prop.getName()).append("=").append(prop.getValue()); + } + return getManager(sb.toString(), factory, new FactoryData(loggerContext, topic, syncSend, properties, key)); + } + + private static class FactoryData { + private final LoggerContext loggerContext; + private final String topic; + private final boolean syncSend; + private final Property[] properties; + private final String key; + + public FactoryData(final LoggerContext loggerContext, final String topic, final boolean syncSend, + final Property[] properties, final String key) { + this.loggerContext = loggerContext; + this.topic = topic; + this.syncSend = syncSend; + this.properties = properties; + this.key = key; + } + + } + + private static class KafkaManagerFactory implements ManagerFactory<KafkaManager, FactoryData> { + @Override + public KafkaManager createManager(String name, FactoryData data) { + return new KafkaManager(data.loggerContext, name, data.topic, data.syncSend, data.properties, data.key); + } + } + } diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractJacksonLayout.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractJacksonLayout.java index 985ef7f..bae404c 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractJacksonLayout.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractJacksonLayout.java @@ -22,19 +22,25 @@ import java.nio.charset.Charset; import java.util.LinkedHashMap; import java.util.Map; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.ThreadContext; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.plugins.PluginBuilderAttribute; import org.apache.logging.log4j.core.config.plugins.PluginElement; -import org.apache.logging.log4j.core.impl.Log4jLogEvent; +import org.apache.logging.log4j.core.impl.ThrowableProxy; import org.apache.logging.log4j.core.jackson.XmlConstants; import org.apache.logging.log4j.core.lookup.StrSubstitutor; +import org.apache.logging.log4j.core.time.Instant; import org.apache.logging.log4j.core.util.KeyValuePair; import org.apache.logging.log4j.core.util.StringBuilderWriter; -import org.apache.logging.log4j.message.ReusableMessage; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.util.ReadOnlyStringMap; import org.apache.logging.log4j.util.Strings; import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonRootName; import com.fasterxml.jackson.annotation.JsonUnwrapped; import com.fasterxml.jackson.core.JsonGenerationException; @@ -274,19 +280,15 @@ abstract class AbstractJacksonLayout extends AbstractStringLayout { } } - private static LogEvent convertMutableToLog4jEvent(final LogEvent event) { - // TODO Jackson-based layouts have certain filters set up for Log4jLogEvent. - // TODO Need to set up the same filters for MutableLogEvent but don't know how... - // This is a workaround. - return event instanceof ReusableMessage ? Log4jLogEvent.createMemento(event) : event; - } - protected Object wrapLogEvent(final LogEvent event) { if (additionalFields.length > 0) { // Construct map for serialization - note that we are intentionally using original LogEvent final Map<String, String> additionalFieldsMap = resolveAdditionalFields(event); // This class combines LogEvent with AdditionalFields during serialization return new LogEventWithAdditionalFields(event, additionalFieldsMap); + } else if (event instanceof Message) { + // If the LogEvent implements the Messagee interface Jackson will not treat is as a LogEvent. + return new ReadOnlyLogEventWrapper(event); } else { // No additional fields, return original object return event; @@ -314,7 +316,7 @@ abstract class AbstractJacksonLayout extends AbstractStringLayout { public void toSerializable(final LogEvent event, final Writer writer) throws JsonGenerationException, JsonMappingException, IOException { - objectWriter.writeValue(writer, wrapLogEvent(convertMutableToLog4jEvent(event))); + objectWriter.writeValue(writer, wrapLogEvent(event)); writer.write(eol); if (includeNullDelimiter) { writer.write('\0'); @@ -358,4 +360,124 @@ abstract class AbstractJacksonLayout extends AbstractStringLayout { this.valueNeedsLookup = AbstractJacksonLayout.valueNeedsLookup(this.value); } } + + private static class ReadOnlyLogEventWrapper implements LogEvent { + + @JsonIgnore + private final LogEvent event; + + public ReadOnlyLogEventWrapper(LogEvent event) { + this.event = event; + } + + @Override + public LogEvent toImmutable() { + return event.toImmutable(); + } + + @Override + public Map<String, String> getContextMap() { + return event.getContextMap(); + } + + @Override + public ReadOnlyStringMap getContextData() { + return event.getContextData(); + } + + @Override + public ThreadContext.ContextStack getContextStack() { + return event.getContextStack(); + } + + @Override + public String getLoggerFqcn() { + return event.getLoggerFqcn(); + } + + @Override + public Level getLevel() { + return event.getLevel(); + } + + @Override + public String getLoggerName() { + return event.getLoggerName(); + } + + @Override + public Marker getMarker() { + return event.getMarker(); + } + + @Override + public Message getMessage() { + return event.getMessage(); + } + + @Override + public long getTimeMillis() { + return event.getTimeMillis(); + } + + @Override + public Instant getInstant() { + return event.getInstant(); + } + + @Override + public StackTraceElement getSource() { + return event.getSource(); + } + + @Override + public String getThreadName() { + return event.getThreadName(); + } + + @Override + public long getThreadId() { + return event.getThreadId(); + } + + @Override + public int getThreadPriority() { + return event.getThreadPriority(); + } + + @Override + public Throwable getThrown() { + return event.getThrown(); + } + + @Override + public ThrowableProxy getThrownProxy() { + return event.getThrownProxy(); + } + + @Override + public boolean isEndOfBatch() { + return event.isEndOfBatch(); + } + + @Override + public boolean isIncludeLocation() { + return event.isIncludeLocation(); + } + + @Override + public void setEndOfBatch(boolean endOfBatch) { + + } + + @Override + public void setIncludeLocation(boolean locationRequired) { + + } + + @Override + public long getNanoTime() { + return event.getNanoTime(); + } + } } diff --git a/log4j-docker/pom.xml b/log4j-docker/pom.xml index f4eeea2..74d7090 100644 --- a/log4j-docker/pom.xml +++ b/log4j-docker/pom.xml @@ -31,6 +31,8 @@ <log4jParentDir>${basedir}/..</log4jParentDir> <docLabel>Log4j Docker Library Documentation</docLabel> <projectDir>/docker</projectDir> + <maven.compiler.source>1.8</maven.compiler.source> + <maven.compiler.target>1.8</maven.compiler.target> <module.name>org.apache.logging.log4j.docker</module.name> </properties> <dependencies> @@ -63,6 +65,25 @@ </dependencies> <build> <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-toolchains-plugin</artifactId> + <version>1.1</version> + <executions> + <execution> + <goals> + <goal>toolchain</goal> + </goals> + </execution> + </executions> + <configuration> + <toolchains> + <jdk> + <version>[8, )</version> + </jdk> + </toolchains> + </configuration> + </plugin> <!-- Include the standard NOTICE and LICENSE --> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/Container.java b/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/Container.java index cb8be45..aee7c51 100644 --- a/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/Container.java +++ b/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/Container.java @@ -31,6 +31,12 @@ public class Container { @JsonProperty("Names") private List<String> names; + @JsonProperty("Path") + private String path; + + @JsonProperty("Args") + private String[] args; + @JsonProperty("Image") private String image; diff --git a/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/IPAMConfig.java b/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/IPAMConfig.java index acc3eeb..b17ad92 100644 --- a/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/IPAMConfig.java +++ b/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/IPAMConfig.java @@ -32,6 +32,9 @@ public class IPAMConfig { @JsonProperty("Gateway") private String gateway; + @JsonProperty("IPv4Address") + private String ipv4Address; + public String getSubnet() { return subnet; } @@ -55,4 +58,12 @@ public class IPAMConfig { public void setGateway(String gateway) { this.gateway = gateway; } + + public String getIpv4Address() { + return ipv4Address; + } + + public void setIpv4Address(String ipv4Address) { + this.ipv4Address = ipv4Address; + } } diff --git a/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/Network.java b/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/Network.java index c542252..5e0674a 100644 --- a/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/Network.java +++ b/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/Network.java @@ -24,13 +24,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class Network { @JsonProperty("IPAMConfig") - private String ipamConfig; + private IPAMConfig ipamConfig; @JsonProperty("Links") private String links; @JsonProperty("Aliases") - private String aliases; + private String[] aliases; @JsonProperty("NetworkID") private String networkId; @@ -62,11 +62,11 @@ public class Network { @JsonProperty("DriverOpts") private String driverOpts; - public String getIpamConfig() { + public IPAMConfig getIpamConfig() { return ipamConfig; } - public void setIpamConfig(String ipamConfig) { + public void setIpamConfig(IPAMConfig ipamConfig) { this.ipamConfig = ipamConfig; } @@ -78,11 +78,11 @@ public class Network { this.links = links; } - public String getAliases() { + public String[] getAliases() { return aliases; } - public void setAliases(String aliases) { + public void setAliases(String[] aliases) { this.aliases = aliases; } diff --git a/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/NetworkSettings.java b/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/NetworkSettings.java index 7dfaa68..a1835d6 100644 --- a/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/NetworkSettings.java +++ b/log4j-docker/src/main/java/org/apache/logging/log4j/docker/model/NetworkSettings.java @@ -28,6 +28,55 @@ public class NetworkSettings { @JsonProperty("Networks") private Map<String, Network> networks; + @JsonProperty("Bridge") + private String bridge; + + @JsonProperty("SandboxID") + private String sandboxId; + + @JsonProperty("HairpinMode") + private boolean hairpinMode; + + @JsonProperty("LinkLocalIPv6Address") + private String linkLocalIPv6Address; + + @JsonProperty("LinkLocalIPv6PrefixLen") + private int linkLocalIPv6PrefixLen; + + @JsonProperty("Ports") + private Map<String, String> ports; + + @JsonProperty("SandboxKey") + private String sandboxKey; + + @JsonProperty("SecondaryIPAddresses") + private String secondaryIPaddresses; + + @JsonProperty("EndpointID") + private String endpointId; + + @JsonProperty("Gateway") + private String gateway; + + @JsonProperty("GlobalIPv6Address") + private String globalIPv6Address; + + @JsonProperty("GlobalIPv6PrefixLen") + private int globalIPv6PrefixLen; + + @JsonProperty("IPAddress") + private String ipAddress; + + @JsonProperty("IPPrefixLen") + private int ipPrefixLen; + + @JsonProperty("IPv6Gateway") + private String ipv6Gateway; + + @JsonProperty("MacAddress") + private String macAddress; + + public Map<String, Network> getNetworks() { return networks; } @@ -35,4 +84,132 @@ public class NetworkSettings { public void setNetworks(Map<String, Network> networks) { this.networks = networks; } + + public String getBridge() { + return bridge; + } + + public void setBridge(String bridge) { + this.bridge = bridge; + } + + public String getSandboxId() { + return sandboxId; + } + + public void setSandboxId(String sandboxId) { + this.sandboxId = sandboxId; + } + + public boolean isHairpinMode() { + return hairpinMode; + } + + public void setHairpinMode(boolean hairpinMode) { + this.hairpinMode = hairpinMode; + } + + public String getLinkLocalIPv6Address() { + return linkLocalIPv6Address; + } + + public void setLinkLocalIPv6Address(String linkLocalIPv6Address) { + this.linkLocalIPv6Address = linkLocalIPv6Address; + } + + public int getLinkLocalIPv6PrefixLen() { + return linkLocalIPv6PrefixLen; + } + + public void setLinkLocalIPv6PrefixLen(int linkLocalIPv6PrefixLen) { + this.linkLocalIPv6PrefixLen = linkLocalIPv6PrefixLen; + } + + public Map<String, String> getPorts() { + return ports; + } + + public void setPorts(Map<String, String> ports) { + this.ports = ports; + } + + public String getSandboxKey() { + return sandboxKey; + } + + public void setSandboxKey(String sandboxKey) { + this.sandboxKey = sandboxKey; + } + + public String getSecondaryIPaddresses() { + return secondaryIPaddresses; + } + + public void setSecondaryIPaddresses(String secondaryIPaddresses) { + this.secondaryIPaddresses = secondaryIPaddresses; + } + + public String getEndpointId() { + return endpointId; + } + + public void setEndpointId(String endpointId) { + this.endpointId = endpointId; + } + + public String getGateway() { + return gateway; + } + + public void setGateway(String gateway) { + this.gateway = gateway; + } + + public String getGlobalIPv6Address() { + return globalIPv6Address; + } + + public void setGlobalIPv6Address(String globalIPv6Address) { + this.globalIPv6Address = globalIPv6Address; + } + + public int getGlobalIPv6PrefixLen() { + return globalIPv6PrefixLen; + } + + public void setGlobalIPv6PrefixLen(int globalIPv6PrefixLen) { + this.globalIPv6PrefixLen = globalIPv6PrefixLen; + } + + public String getIpAddress() { + return ipAddress; + } + + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + public int getIpPrefixLen() { + return ipPrefixLen; + } + + public void setIpPrefixLen(int ipPrefixLen) { + this.ipPrefixLen = ipPrefixLen; + } + + public String getIpv6Gateway() { + return ipv6Gateway; + } + + public void setIpv6Gateway(String ipv6Gateway) { + this.ipv6Gateway = ipv6Gateway; + } + + public String getMacAddress() { + return macAddress; + } + + public void setMacAddress(String macAddress) { + this.macAddress = macAddress; + } } diff --git a/log4j-docker/src/site/markdown/index.md.vm b/log4j-docker/src/site/markdown/index.md.vm new file mode 100644 index 0000000..42ee2cc --- /dev/null +++ b/log4j-docker/src/site/markdown/index.md.vm @@ -0,0 +1,55 @@ +<!-- vim: set syn=markdown : --> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +#set($dollar = '$') +#set($h1='#') +#set($h2='##') + +$h1 Log4j Docker Support + +Log4j supports Docker by providing a Lookup to retrieve container information. + +$h2 Accessing Docker + +The Log4j Docker support requires access to the Docker REST intrerface. In practical terms this means the +application either needs access to unix:///var/run/docker.sock through a volume mount (not recommended), +bind Docker to another host/port or unix socket. or use a proxy application to provide access. The +Log4j Spring Cload sample application uses a socat proxy to access Docker. + +$h2 Lookup Attributes + +Log4j Docker provides access to the following container attributes: + +* containerId - The full id assigned to the container. +* containerName - The name assigned to the container. +* imageId - The id assigned to the image. +* imageName - The name assigned to the image. +* shortContainerId - The first 12 characters of the container id. +* shortImageId - The first 12 characters of the image id. + +#set( $D = '${' ) +#set( $container = 'docker:containerId}') +Attributes may be accessed by adding +``` +$D$container +``` +to the configuration. Note that docker variables are only resolved once during logging initializaton so they +shouldn't be referenced with more than one '$' character. + +$h2 Requirements +Log4j Docker requires Log4j Core, Log4j API and a minimum of Java 8. +For more information, see [Runtime Dependencies](../runtime-dependencies.html). diff --git a/log4j-docker/src/site/site.xml b/log4j-docker/src/site/site.xml new file mode 100644 index 0000000..7322f3b --- /dev/null +++ b/log4j-docker/src/site/site.xml @@ -0,0 +1,52 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project name="Log4j Docker Support" + xmlns="http://maven.apache.org/DECORATION/1.4.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/DECORATION/1.4.0 http://maven.apache.org/xsd/decoration-1.4.0.xsd";> + <body> + <links> + <item name="Apache" href="http://www.apache.org/"; /> + <item name="Logging Services" href="http://logging.apache.org/"/> + <item name="Log4j" href="../index.html"/> + </links> + + <!-- Component-specific reports --> + <menu ref="reports"/> + + <!-- Overall Project Info --> + <menu name="Log4j Project Information" img="icon-info-sign"> + <item name="Dependencies" href="../dependencies.html" /> + <item name="Dependency Convergence" href="../dependency-convergence.html" /> + <item name="Dependency Management" href="../dependency-management.html" /> + <item name="Project Team" href="../team-list.html" /> + <item name="Mailing Lists" href="../mail-lists.html" /> + <item name="Issue Tracking" href="../issue-tracking.html" /> + <item name="Project License" href="../license.html" /> + <item name="Source Repository" href="../source-repository.html" /> + <item name="Project Summary" href="../project-summary.html" /> + </menu> + + <menu name="Log4j Project Reports" img="icon-cog"> + <item name="Changes Report" href="../changes-report.html" /> + <item name="JIRA Report" href="../jira-report.html" /> + <item name="Surefire Report" href="../surefire-report.html" /> + <item name="RAT Report" href="../rat-report.html" /> + </menu> + </body> +</project> diff --git a/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/BatchEvent.java b/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/BatchEvent.java index 41af4fa..9a6f86d 100644 --- a/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/BatchEvent.java +++ b/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/BatchEvent.java @@ -35,4 +35,8 @@ public class BatchEvent { public List<Event> getEvents() { return events; } + + public int size() { + return events.size(); + } } diff --git a/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/FlumeAppender.java b/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/FlumeAppender.java index 210aadd..d99cbf7 100644 --- a/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/FlumeAppender.java +++ b/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/FlumeAppender.java @@ -19,7 +19,6 @@ package org.apache.logging.log4j.flume.appender; import java.io.Serializable; import java.util.Locale; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import org.apache.logging.log4j.core.Appender; import org.apache.logging.log4j.core.Filter; diff --git a/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/FlumeAvroManager.java b/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/FlumeAvroManager.java index c1dd288..d1767e5 100644 --- a/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/FlumeAvroManager.java +++ b/log4j-flume-ng/src/main/java/org/apache/logging/log4j/flume/appender/FlumeAvroManager.java @@ -16,6 +16,7 @@ */ package org.apache.logging.log4j.flume.appender; +import java.util.List; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -24,6 +25,7 @@ import org.apache.flume.api.RpcClient; import org.apache.flume.api.RpcClientFactory; import org.apache.logging.log4j.core.appender.AppenderLoggingException; import org.apache.logging.log4j.core.appender.ManagerFactory; +import org.apache.logging.log4j.util.Timer; /** * Manager for FlumeAvroAppenders. @@ -50,7 +52,7 @@ public class FlumeAvroManager extends AbstractFlumeManager { private final int current = 0; - private RpcClient rpcClient = null; + private volatile RpcClient rpcClient = null; private BatchEvent batchEvent = new BatchEvent(); private long nextSend = 0; @@ -97,9 +99,9 @@ public class FlumeAvroManager extends AbstractFlumeManager { if (batchSize <= 0) { batchSize = 1; - } - - final StringBuilder sb = new StringBuilder("FlumeAvro["); + }; + final StringBuilder sb = new StringBuilder(name); + sb.append(" FlumeAvro["); boolean first = true; for (final Agent agent : agents) { if (!first) { @@ -149,9 +151,13 @@ public class FlumeAvroManager extends AbstractFlumeManager { return delayMillis; } - public synchronized void send(final BatchEvent events) { + public void send(final BatchEvent events) { if (rpcClient == null) { - rpcClient = connect(agents, retries, connectTimeoutMillis, requestTimeoutMillis); + synchronized (this) { + if (rpcClient == null) { + rpcClient = connect(agents, retries, connectTimeoutMillis, requestTimeoutMillis); + } + } } if (rpcClient != null) { @@ -175,7 +181,7 @@ public class FlumeAvroManager extends AbstractFlumeManager { } @Override - public synchronized void send(final Event event) { + public void send(final Event event) { if (batchSize == 1) { if (rpcClient == null) { rpcClient = connect(agents, retries, connectTimeoutMillis, requestTimeoutMillis); @@ -199,14 +205,22 @@ public class FlumeAvroManager extends AbstractFlumeManager { throw new AppenderLoggingException("No Flume agents are available"); } } else { - batchEvent.addEvent(event); - final int eventCount = batchEvent.getEvents().size(); - if (eventCount == 1) { - nextSend = System.nanoTime() + delayNanos; + int eventCount; + BatchEvent batch = null; + synchronized(batchEvent) { + batchEvent.addEvent(event); + eventCount = batchEvent.size(); + long now = System.nanoTime(); + if (eventCount == 1) { + nextSend = now + delayNanos; + } + if (eventCount >= batchSize || now >= nextSend) { + batch = batchEvent; + batchEvent = new BatchEvent(); + } } - if (eventCount >= batchSize || System.nanoTime() >= nextSend) { - send(batchEvent); - batchEvent = new BatchEvent(); + if (batch != null) { + send(batch); } } } diff --git a/log4j-samples/log4j-samples-flume-embedded/pom.xml b/log4j-samples/log4j-samples-flume-embedded/pom.xml index 43982a1..c1895b3 100644 --- a/log4j-samples/log4j-samples-flume-embedded/pom.xml +++ b/log4j-samples/log4j-samples-flume-embedded/pom.xml @@ -28,6 +28,7 @@ <url>http://maven.apache.org</url> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <deploy.plugin.version>2.8.2</deploy.plugin.version> </properties> <dependencies> <dependency> diff --git a/log4j-samples/log4j-samples-flume-remote/pom.xml b/log4j-samples/log4j-samples-flume-remote/pom.xml index 8f33337..228bcec 100644 --- a/log4j-samples/log4j-samples-flume-remote/pom.xml +++ b/log4j-samples/log4j-samples-flume-remote/pom.xml @@ -28,6 +28,7 @@ <url>http://maven.apache.org</url> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <deploy.plugin.version>2.8.2</deploy.plugin.version> </properties> <dependencies> <dependency> diff --git a/log4j-spring-cloud-config/log4j-spring-cloud-config-client/pom.xml b/log4j-spring-cloud-config/log4j-spring-cloud-config-client/pom.xml index 6ab556b..1317ced 100644 --- a/log4j-spring-cloud-config/log4j-spring-cloud-config-client/pom.xml +++ b/log4j-spring-cloud-config/log4j-spring-cloud-config-client/pom.xml @@ -26,9 +26,9 @@ <artifactId>log4j-spring-cloud-config-client</artifactId> <packaging>jar</packaging> <name>Apache Log4j Spring Cloud Config Client Support</name> - <description></description> + <description>Apache Log4j Spring Cloud Config Client Support</description> <properties> - <log4jParentDir>${basedir}/..</log4jParentDir> + <log4jParentDir>${basedir}/../..</log4jParentDir> <docLabel>Log4j Spring Cloud Config Client Documentation</docLabel> <projectDir>/log4j-spring-cloud-config-client</projectDir> <module.name>org.apache.logging.log4j.spring.cloud.config.client</module.name> diff --git a/log4j-spring-cloud-config/log4j-spring-cloud-config-client/src/site/markdown/index.md b/log4j-spring-cloud-config/log4j-spring-cloud-config-client/src/site/markdown/index.md new file mode 100644 index 0000000..5530859 --- /dev/null +++ b/log4j-spring-cloud-config/log4j-spring-cloud-config-client/src/site/markdown/index.md @@ -0,0 +1,88 @@ +<!-- vim: set syn=markdown : --> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Log4j Spring Cloud Configuration + +This module allows logging configuration files managed in be dynamically updated when new versions are available in +Spring Cloud Configuration. + +## Overview + +Spring Boot applications initialize logging 3 times. +1. SpringApplication declares a Logger. This Logger will be initialized using Log4j's "normal" mechanisms. Thus +a system property named log4j2.configurationFile will be checked to see if a specific configuration file has been +provided, otherwise it will search for a configuration file on the classpath. The property may also be declare +in log4j2.component.properties. + +## Usage + +Log4j configuration files that specify a monitor interval of greater than zero will use polling to determine +whether the configuration has been updated. If the monitor interval is zero then Log4j will listen for notifications +from Spring Cloud Config and will check for configuration changes each time an event is generated. If the +monitor interval is less than zero Log4j will not check for changes to the logging configuration. + +When referencing a configuration located in Spring Cloud Config the configuration should be referenced similar to + +``` +log4j.configurationFile=http://host.docker.internal:8888/ConfigService/sampleapp/default/master/log4j2.xml +``` +When running in a docker container host.docker.internal may be used as the domain name to access an application +running on the same hose outside of the docker container. Note that in accordance with Spring Cloud Config +practices but the application, profile, and label should be specified in the url. + +The Spring Cloud Config support also allows connections using TLS and/or basic authentication. When using basic +authentication the userid and password may be specified as system properties or in log4j2.component.properties as + +``` +log4j2.configurationUserName=guest +log4j2.configurationPassword=guest +``` +Note that Log4j currently does not support encrypting the password. + +If more extensive authentication is required an ```AuthorizationProvider``` can be implemented and defined in +the log4j2.authorizationProvider system property or in log4j2.component.properties. + +TLS can be enabled by adding the following system properties or defining them in log4j2.component.properties + +| Property | Optional or Default Value | Description | +| ------------- |-------|:-------------| +| log4j2.trustStore.location | Optional | The location of the trust store. If not provided the default trust store will be used.| +| log4j2.trustStore.password | Optional | Password needed to access the trust store. | +| log4j2.trustStore.passwordFile | Optinoal | The location of a file that contains the password for the trust store. | +| log4j2.trustStore.passwordEnvironmentVariable | Optional | The name of the environment variable that contains the trust store password. | +| log4j2.trustStore.keyStoreType | Required if keystore location provided | The type of key store. | +| log4j2.trustStore.keyManagerFactoryAlgorithm | Optional | Java cryptographic algorithm. | +| log4j2.keyStore.location | Optional | The location of the key store. If not provided the default key store will be used.| +| log4j2.keyStore.password | Optional | Password needed to access the key store. | +| log4j2.keyStore.passwordFile | Optional | The location of a file that contains the password for the key store. | +| log4j2.keyStore.passwordEnvironmentVariable | Optional | The name of the environment variable that contains the key store password.| +| log4j2.keyStore.type | Required if trust store location provided. | The type of key store. | +| log4j2.keyStore.keyManagerFactoryAlgorithm | Optional | Java cryptographic algorithm. | +| log4j2.ssl.verifyHostName | false | true or false | + + + +## Requirements + +The Log4j 2 Spring Cloud Configuration integration has a dependency on Log4j 2 API, Log4j 2 Core, and +Spring Cloud Configuration versions 2.0.3.RELEASE or 2.1.1.RELEASE or later versions it either release series. +For more information, see [Runtime Dependencies](../../runtime-dependencies.html). + + + + diff --git a/log4j-spring-cloud-config/log4j-spring-cloud-config-client/src/site/site.xml b/log4j-spring-cloud-config/log4j-spring-cloud-config-client/src/site/site.xml new file mode 100644 index 0000000..5abfffd --- /dev/null +++ b/log4j-spring-cloud-config/log4j-spring-cloud-config-client/src/site/site.xml @@ -0,0 +1,52 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project name="Log4j Spring Cloud Config Integration" + xmlns="http://maven.apache.org/DECORATION/1.4.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/DECORATION/1.4.0 http://maven.apache.org/xsd/decoration-1.4.0.xsd";> + <body> + <links> + <item name="Apache" href="http://www.apache.org/"; /> + <item name="Logging Services" href="http://logging.apache.org/"/> + <item name="Log4j" href="../index.html"/> + </links> + + <!-- Component-specific reports --> + <menu ref="reports"/> + + <!-- Overall Project Info --> + <menu name="Log4j Project Information" img="icon-info-sign"> + <item name="Dependencies" href="../dependencies.html" /> + <item name="Dependency Convergence" href="../dependency-convergence.html" /> + <item name="Dependency Management" href="../dependency-management.html" /> + <item name="Project Team" href="../team-list.html" /> + <item name="Mailing Lists" href="../mail-lists.html" /> + <item name="Issue Tracking" href="../issue-tracking.html" /> + <item name="Project License" href="../license.html" /> + <item name="Source Repository" href="../source-repository.html" /> + <item name="Project Summary" href="../project-summary.html" /> + </menu> + + <menu name="Log4j Project Reports" img="icon-cog"> + <item name="Changes Report" href="../changes-report.html" /> + <item name="JIRA Report" href="../jira-report.html" /> + <item name="Surefire Report" href="../surefire-report.html" /> + <item name="RAT Report" href="../rat-report.html" /> + </menu> + </body> +</project> diff --git a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/README.md b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/README.md index 93448e3..3251015 100644 --- a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/README.md +++ b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/README.md @@ -1,53 +1,33 @@ -#Local Development -###Prerequisites -Note: This guide uses Homebrew (package manage for macOS). It is not necessary to use Homebrew, but it does -simplify the installation process. -* Install Homebrew: `/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"` - * If you already have Homebrew installed, make sure it is updated: `brew update` -* Add the cask-versions tap for Homebrew: `brew tap homebrew/cask-versions` -* Install Docker: `brew cask install docker` -* Install docker-machine: `brew install docker-machine` -* Set up Docker connection to nexus: https://confluence.nextiva.xyz/display/DP/Docker+connection+to+nexus -* Find OpenJDK 11 cask `brew search java` - * Currently the cask for java 11 is just named 'java', but this may change in the future. You can verify by - running `brew cask info <caskName>` to verify the version. -* Install OpenJDK 11 cask: `brew cask install java` -* Set JAVA_HOME to java 11 installation directory (/Library/Java/JavaVirtualMachines/jdk-11.0.1.jdk/Contents/Home) - * If you need to use and switch between multiple Java versions, consider using jEnv to simplify this process - http://www.jenv.be +##Log4j Spring Cloud Sample Application -###Starting the Application -* Start local postgres image `./docker/up.sh` -* Compile and start local application image `./docker/restartApp.sh` - * Alternatively: Run FulfillmentApplication.java as a Spring Boot application using java -jar target/fulfillment-service.jar. -* Local swagger URL is available at http://localhost:8080/swagger-ui.html +This application uses Spring Boot and reads the logging configuration from the companion Spring Cloud Config Server +project. The log4j2.xml file is located in the config-repo directory in that project. -# Java client -* Using the fulfillment-service-client is recommended when integrating Java applications with this service. -###Using the client -* Add the latest version of fulfillment-service-client as a dependency to your application. -* Import the OrderSubmissionServiceClient class into your application configuration. -* Specify fulfillment-service.url as a property +This sample packages the application in a docker container that is packaged with rabbit-mq (to allow dynamic updates +from Spring Cloud Config), fluent-bit (to test as a log forwarder), and Apache Flume (to test as a log forwarder). +###Prerequisites +Note: This guide assumes you already have docker installed. If you do not you may either use homebrew to install +it or follow the instructions at https://docs.docker.com/docker-for-mac/install/. -# Database setup -Docker will create a container for postgres - a local version of Postgresql does not need to be installed. -Before starting the application the tables in Postgres must be created. Until this is automatced login to pgAdmin -using fulfillment_app/fulfillment_app as the credentialsand run the script in -fulfillment-service-web/resources/postgres/schema.sql. +Like Log4j, the sample app uses the Maven toolchains plugin. The sample app may be built with Java 8 but is +configured to run in a docker container with Java 11. -# Environment properties -* This applications uses Kubernetes Config Maps to configure properties for an environment. The properties -configured in application.yml will be used unless overridden at https://git.nextiva.xyz/projects/REL/repos/k8s-platform/browse +The KafkaAppender requires a Kafka instance to write to. On MacOS a Kafka instance can be created by +``` +brew install kafka +zookeeper-server-start /usr/local/etc/kafka/zookeeper.properties & kafka-server-start /usr/local/etc/kafka/server.properties +``` + +###Starting the Application +* Start the companion rabbit-mq, fluent-bit and flume images `./docker/up.sh` +* Compile and start local application image `./docker/restartApp.sh` +* The application exposes two endpoints. + 1. http://localhost:8080/sample/log?threads=1&count=100000 This endpoint will log up to 100,000 events using + 1 or more threads. + 1. http://localhost:8080/sample/exception This endpoint generates an exception that can be used to verify whether + multiline logging works with the chosen set of components. -# Swagger -* Dev: https://fulfillment-service.dev.nextiva.io/swagger-ui.ml -* Rc: https://fulfillment-service.qa.nextiva.io/swagger-ui.html -* Prod: https://fulfillment-service.prod.nextiva.io/swagger-ui.html +###Viewing the logs -#Automated Testing -<!--- -* Unit tests can be run using `mvn test -Dgroups=UnitTest` -* Integration tests (do not require Fulfillment Service but do require connection to third parties) -can be run using `mvn test -Dgroups=IntegrationTest` ---> -* Functional tests (those that require Fulfillment Service running) can be run using `mvn -P integration-tests verify` \ No newline at end of file +Accessing the log files varies depending on the appending being used. + diff --git a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/docker/app-compose.yml b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/docker/app-compose.yml new file mode 100755 index 0000000..01cb293 --- /dev/null +++ b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/docker/app-compose.yml @@ -0,0 +1,21 @@ +version: "3" +services: + sampleapp: + container_name: sampleapp + image: sampleapp + environment: + DOCKER_URI: http://socat:1234 + SERVICE_PARAMS: --spring.config.location=classpath:/,classpath:/application-local-docker.yml + ports: + - "5005:5005" + - "8080:4567" + networks: + sample_network: + aliases: + - sampleapp + +networks: + sample_network: + +volumes: + pgdata: \ No newline at end of file diff --git a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/docker/docker-compose.yml b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/docker/docker-compose.yml index 10a5b13..76a0d09 100755 --- a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/docker/docker-compose.yml +++ b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/docker/docker-compose.yml @@ -65,6 +65,7 @@ services: aliases: - flume + networks: sample_network: diff --git a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/pom.xml b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/pom.xml index 0122205..f059876 100644 --- a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/pom.xml +++ b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-application/pom.xml @@ -18,6 +18,8 @@ <url>http://maven.apache.org</url> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <maven.compiler.source>1.8</maven.compiler.source> + <maven.compiler.target>1.8</maven.compiler.target> <!--<manifestfile>${project.build.outputDirectory}/META-INF/MANIFEST.MF</manifestfile>--> </properties> @@ -114,6 +116,11 @@ <version>1.9.0</version> </dependency> <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + <version>2.2.0</version> + </dependency> + <dependency> <groupId>javax.servlet</groupId> <artifactId>javax.servlet-api</artifactId> <version>3.1.0</version> @@ -131,6 +138,15 @@ <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-site-plugin</artifactId> + <version>${site.plugin.version}</version> + <configuration> + <skip>true</skip> + <skipDeploy>true</skipDeploy> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-toolchains-plugin</artifactId> <version>1.1</version> <executions> diff --git a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/pom.xml b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/pom.xml index 684b0cf..900015b 100644 --- a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/pom.xml +++ b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/pom.xml @@ -36,7 +36,7 @@ <maven.google.code.findbugs.findbugs.version>3.0.1</maven.google.code.findbugs.findbugs.version> <maven.jacoco.version>0.8.1</maven.jacoco.version> <maven.pmd.version>3.9.0</maven.pmd.version> - + <site.plugin.version>3.4</site.plugin.version> <!-- maven plugin config --> <pmd.failurePriority>2</pmd.failurePriority> @@ -120,6 +120,15 @@ <build> <plugins> <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-site-plugin</artifactId> + <version>${site.plugin.version}</version> + <configuration> + <skip>true</skip> + <skipDeploy>true</skipDeploy> + </configuration> + </plugin> + <plugin> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-maven-plugin</artifactId> </plugin> @@ -177,6 +186,15 @@ </plugins> </build> + <distributionManagement> + <downloadUrl>https://logging.apache.org/log4j/2.x/download.html</downloadUrl> + <!-- site is only included to make maven-site-plugin stop complaining --> + <site> + <id>www.example.com</id> + <url>scp://www.example.com/www/docs/project/</url> + </site> + </distributionManagement> + <repositories> <repository> <id>spring-snapshots</id> diff --git a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/src/main/config-repo/log4j2.xml b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/src/main/config-repo/log4j2.xml index 1c061a7..1e5eea6 100644 --- a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/src/main/config-repo/log4j2.xml +++ b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/src/main/config-repo/log4j2.xml @@ -22,7 +22,7 @@ <KeyValuePair key="imageName" value="\${docker:imageName}"/> </JsonLayout> </Flume> - <Flume name="avro" ignoreExceptions="false" batchSize="100" compress="false"> + <Flume name="avro" ignoreExceptions="false" batchSize="1" compress="false"> <Agent host="flume" port="5050"/> <JsonLayout properties="true" compact="true" eventEol="true"> <KeyValuePair key="containerId" value="\${docker:containerId}"/> @@ -30,6 +30,25 @@ <KeyValuePair key="imageName" value="\${docker:imageName}"/> </JsonLayout> </Flume> + <Flume name="avroSyslog" ignoreExceptions="false" batchSize="100" compress="false"> + <Agent host="flume" port="5050"/> + <RFC5424Layout enterpriseNumber="18060" includeMDC="true" mdcId="RequestContext" appName="Sample" + mdcPrefix=""> + <LoggerFields> + <KeyValuePair key="containerId" value="\${docker:containerId}"/> + <KeyValuePair key="containerName" value="\${docker:containerName}"/> + <KeyValuePair key="imageName" value="\${docker:imageName}"/> + </LoggerFields> + </RFC5424Layout> + </Flume> + <Kafka name="Kafka" topic="log-test" syncSend="false"> + <JsonLayout properties="true" compact="true" eventEol="true"> + <KeyValuePair key="containerId" value="\${docker:containerId}"/> + <KeyValuePair key="containerName" value="\${docker:containerName}"/> + <KeyValuePair key="imageName" value="\${docker:imageName}"/> + </JsonLayout> + <Property name="bootstrap.servers">host.docker.internal:9092</Property> + </Kafka> <Socket name="fluent-bit" host="fluent-bit" port="24221"> <JsonLayout properties="true" compact="true" eventEol="true"> <KeyValuePair key="containerId" value="\${docker:containerId}"/> @@ -37,9 +56,34 @@ <KeyValuePair key="imageName" value="\${docker:imageName}"/> </JsonLayout> </Socket> - + <RollingFile name="RollingFile" fileName="/var/log/sampleapp/app.log" + filePattern="/var/log/sampleapp/archive/app.log.%i"> + <JsonLayout properties="true" compact="true" eventEol="true"> + <KeyValuePair key="containerId" value="\${docker:containerId}"/> + <KeyValuePair key="containerName" value="\${docker:containerName}"/> + <KeyValuePair key="imageName" value="\${docker:imageName}"/> + </JsonLayout> + <SizeBasedTriggeringPolicy size="10MB" /> + <DefaultRolloverStrategy max="5"/> + </RollingFile> + <RollingFile name="RollingSyslog" fileName="/var/log/sampleapp/syslog.log" + filePattern="/var/log/sampleapp/archive/syslog.log.%i"> + <RFC5424Layout enterpriseNumber="18060" includeMDC="true" mdcId="RequestContext" appName="Sample" + mdcPrefix=""> + <LoggerFields> + <KeyValuePair key="containerId" value="\${docker:containerId}"/> + <KeyValuePair key="containerName" value="\${docker:containerName}"/> + <KeyValuePair key="imageName" value="\${docker:imageName}"/> + </LoggerFields> + </RFC5424Layout> + <SizeBasedTriggeringPolicy size="10MB" /> + <DefaultRolloverStrategy max="5"/> + </RollingFile> </Appenders> <Loggers> + <Logger name="org.apache.kafka" level="warn" additivity="false"> + <AppenderRef ref="console"/> + </Logger> <Logger name="org.apache.flume" level="warn" additivity="false"> <AppenderRef ref="console"/> </Logger> @@ -47,10 +91,7 @@ <AppenderRef ref="console"/> </Logger> <Root level="DEBUG"> - <AppenderRef ref="flume"/> - <!--<AppenderRef ref="flume"/>--> - <!--<AppenderRef ref="console"/>--> - <!--<AppenderRef ref="fluent-bit"/>--> + <AppenderRef ref="Kafka"/> </Root> </Loggers> </Configuration> \ No newline at end of file diff --git a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/src/main/resources/log4j2.xml b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/src/main/resources/log4j2.xml index ff66652..3a01958 100644 --- a/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/src/main/resources/log4j2.xml +++ b/log4j-spring-cloud-config/log4j-spring-cloud-config-samples/log4j-spring-cloud-config-sample-server/src/main/resources/log4j2.xml @@ -22,7 +22,7 @@ </Console> </Appenders> <Loggers> - <Root level="debug"> + <Root level="ERROR"> <AppenderRef ref="STDOUT" /> </Root> </Loggers> diff --git a/log4j-spring-cloud-config/pom.xml b/log4j-spring-cloud-config/pom.xml index a89c6ed..56e0d40 100644 --- a/log4j-spring-cloud-config/pom.xml +++ b/log4j-spring-cloud-config/pom.xml @@ -34,6 +34,7 @@ <spring-cloud-version>Finchley.SR3</spring-cloud-version> <spring-boot.version>2.0.8.RELEASE</spring-boot.version> <springVersion>5.0.12.RELEASE</springVersion> + <log4jParentDir>${basedir}/..</log4jParentDir> </properties> <dependencyManagement> <dependencies> diff --git a/log4j-spring-cloud-config/src/site/markdown/index.md b/log4j-spring-cloud-config/src/site/markdown/index.md new file mode 100644 index 0000000..7f51be4 --- /dev/null +++ b/log4j-spring-cloud-config/src/site/markdown/index.md @@ -0,0 +1,21 @@ +<!-- vim: set syn=markdown : --> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Log4j Spring Cloud Config + + diff --git a/log4j-spring-cloud-config/src/site/site.xml b/log4j-spring-cloud-config/src/site/site.xml new file mode 100644 index 0000000..14bbb87 --- /dev/null +++ b/log4j-spring-cloud-config/src/site/site.xml @@ -0,0 +1,52 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project name="Log4j Spring Cloud Config" + xmlns="http://maven.apache.org/DECORATION/1.4.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/DECORATION/1.4.0 http://maven.apache.org/xsd/decoration-1.4.0.xsd";> + <body> + <links> + <item name="Apache" href="http://www.apache.org/"; /> + <item name="Logging Services" href="http://logging.apache.org/"/> + <item name="Log4j" href="../index.html"/> + </links> + + <!-- Component-specific reports --> + <menu ref="reports"/> + + <!-- Overall Project Info --> + <menu name="Log4j Project Information" img="icon-info-sign"> + <item name="Dependencies" href="../dependencies.html" /> + <item name="Dependency Convergence" href="../dependency-convergence.html" /> + <item name="Dependency Management" href="../dependency-management.html" /> + <item name="Project Team" href="../team-list.html" /> + <item name="Mailing Lists" href="../mail-lists.html" /> + <item name="Issue Tracking" href="../issue-tracking.html" /> + <item name="Project License" href="../license.html" /> + <item name="Source Repository" href="../source-repository.html" /> + <item name="Project Summary" href="../project-summary.html" /> + </menu> + + <menu name="Log4j Project Reports" img="icon-cog"> + <item name="Changes Report" href="../changes-report.html" /> + <item name="JIRA Report" href="../jira-report.html" /> + <item name="Surefire Report" href="../surefire-report.html" /> + <item name="RAT Report" href="../rat-report.html" /> + </menu> + </body> +</project> diff --git a/src/site/markdown/index.md.vm b/src/site/markdown/index.md.vm index 36a42ce..7c94a85 100644 --- a/src/site/markdown/index.md.vm +++ b/src/site/markdown/index.md.vm @@ -107,6 +107,12 @@ $h3 Integrating with Application Servers Version 2.10.0 introduces a the module log4j-appserver to improve integration with Apache Tomcat and Eclipse Jetty. +$h3 Cloud Enabled + +Version 2.12.0 introduces support for accessing Dcoker container information via a Lookup and for accessing +and updating the Log4j configuration through Spring Clooud Configuration. See [Logging in the Cloud](manual/cloud.html) +for details. + $h2 Documentation The Log4j 2 User's Guide is available on this [site](manual/index.html) or as a downloadable diff --git a/src/site/markdown/manual/cloud.md b/src/site/markdown/manual/cloud.md new file mode 100644 index 0000000..71a1125 --- /dev/null +++ b/src/site/markdown/manual/cloud.md @@ -0,0 +1,212 @@ +<!-- vim: set syn=markdown : --> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Using Log4j in Cloud Enabled Applications + +## The Twelve-Factor Application + +The Logging Guidelines for [The Twelve-Factor App](https://12factor.net/logs) state the all logs should be routed +unbuffered to stdout. Since this is the least common denominator it is guaranteed to work for all applications. Howeever, +as with any set of general guidelines, choosing the least common denominator approach comes at a cost. Some of the costs +in Java applications include: + +1. Java stack traces are multi-line log messages. The standard docker log driver cannot handle these properly. See +[Docker Issue #22920](https://github.com/moby/moby/issues/22920) which was closed with the message "Don't Care". +1. When logging to stdout in Docker, log events pass through Java's standard output handling which is then directed +to the operating system so that the output can be piped into a file. The overhead of all this is measurably slower +than just writing directly to a file as can be seen by the performance results below where logging +to stdout is anywhere from 20 to 200% slower than logging directly to the file. However, these results alone +would not be enough to argue against writing to the standard output stream as they only amount to about 20-30 +microseconds per logging call. +1. When performing audit logging using a framework such as log4j-audit guaranteed delivery of the audit events +is required. Many of the options for writing the output, including writing to the standard output stream, do +not guarantee delivery. In these cases the event must be delivered to a "forwarder" that acknowledges receipt +only when it has placed the event in durable storage, such as what Apache Flume or Apache Kafka will do. + +## Logging Approaches + +All the solutions discussed on this page are predicated with the idea that log files cannot permanently +reside on the file system and that all log events should be routed to one or more log analysis tools that will +be used for reporting and alreting. There are many ways to forward and collect events to be sent to the +log analysis tools. + +Note that any approach that bypasses Docker's logging drivers requires Log4j's +[Docker Loookup](lookups.html#DockerLookup) to allow Docker attributes to be injected into the log events. + +### Logging to the Standard Output Stream + +As discussed above, this is the recommended 12-Factor approach for applications running in a docker container. +The Log4j team does not recommend this approach if exceptions will be logged by the Java application. + + + +### Logging to the Standard Output Stream with the Docker Fluentd Logging Driver + +Docker provides alternate [logging drivers](https://docs.docker.com/config/containers/logging/configure/), +such as [gelf](https://docs.docker.com/config/containers/logging/gelf/) or +[fluentd](https://docs.docker.com/config/containers/logging/fluentd/), that +can be used to redirect the standard output stream to a log forwarder or log aggregator. + +When routing to a log forwarder it is expected that the forwarder will have the same lifetime as the +application. If the forwarder should fail the management tools would be expected to also terminate +other containers dependent on the forwarder. + + + +As an alternative the logging drivers could be configured to route events directly to a logging aggregator. +This is generally not a good idea as the logging drivers only allow a single host and port to be configured. +The docker documentation isn't clear but infers that log events will be dropped when log events cannot be +delivered so this method should not be used if a highly available solution is required. + + + +### Logging to a File + +While this is not the recommended 12-Factor approach, it performs very well. However, it requires that the +application declare a volume where the log files will reside and then configure the log forwarder to tail +those files. Care must also be taken to automatically manage the disk space used for the logs, which Log4j +can perform via the Delete action on the [RollingFileAppender](appenders.html#RollingFileAppender). + + + +### Sending Directly to a Log Forwarder via TCP + +Sending logs directly to a Log Forwarder is simple as it generally just requires that the forwarder's +host and port be configured on a SocketAppender with an appropriate layout. + + + +### Sending Directly to a Log Aggregator via TCP + +Similar to sending logs to a forwarder, logs can also be sent to a cluster of aggregators. However, +setting this up is not as simple. Since, to be highly available, a cluster of aggregators must be used. +However, the SocketAppender currently can only be configured with a single host and port. To allow +for failover if the primary aggregator fails the SocketAppender must be enclosed in a +[FailoverAppender](appenders.html#FailoverAppender), +which would also have the secondary aggregator configured. + + + +## Managing Logging Configuration + +Spring Boot provides another least common denominator approach to logging configuration. It will let you set the +log level for various Loggers within an application which can be dynamically updated via REST endpoints provided +by Spring. While this works in a lot of cases it does not support any of the more advanced filtering featurs of +Log4j. For example, since it cannot add or modify any Filters other than the log level of a logger, changes cannot be made to allow +all log events for a specific user or customer to temporarily be logged +(see [DynamicThresholdFilter](filters.html#DynamicThresholdFilter) or +[ThreadContextMapFilter](filters.html#ThreadContextMapFilter)) or any other kinds of changes to filters. +Also, in a micro-services, clustered environment it is quite likely that these changes will need to be propagated +to multiple servers at the same time. Trying to achieve this via REST calls could be difficult. + +Log4j supports dynamic reconfiguration. since the first release Log4j has supported reconfiguration through a file. +Beginning with Log4j 2.12.0 Log4j also supports accessing the configuration via HTTP(S) and monitoring the file +for changes by using the HTTP "If-Modified-Since" header. A patch has also been integrated into Spring Cloud Config +starting with versions 2.0.3 and 2.1.1 for it to honor the If-Modified-Since header. In addition, the +log4j-spring-cloud-config project will listen for update events published by Spring Cloud Bus and then verify +that the configuratoin file has been modified, so polling via HTTP is not required. + +Log4j also supports composite configurations. A distributed application spread across microservices could +share a common configuration file that could be used to control things like enabling debug logging for a +specific user. + +While the standard Spring Boot REST endpoints to update logging will still work any changes made by those +REST endpoints will be lost if Log4j reconfigures itself do to changes in the logging configuration file. + +Further information regarding integration of the log4j-spring-cloud-config-client can be found at +[Log4j Spring Cloud Config Client](../log4j-spring-cloud-config/log4j-spring-cloud-config-client/index.html) + +## Integration with Docker + +Applications within a Docker container that log using a Docker logging driver can include special +attributes in the formatted log event as described at +[Customize Log Driver Output](https://docs.docker.com/config/containers/logging/log_tags/). Log4j +provides similar functionality via the [Docker Loookup](lookups.html#DockerLookup). More information on +Log4j's Docker support may also be found at [Log4j-Docker](../log4j-docker/index.html). + +## Appender Performance +The numbers in the table below represent how much time was required for the application to call logger.debug +100,000 times. These numbers only include the time taken to deliver to the specifcly noted endpoint and +many not include the actual time required before they are availble for viewing. All measurements were +performed on a MacBook Pro with a 2.9GHz Intel Core I9 processor with 6 physical and 12 logical cores, +32GB of 2400 MHz DDR4 RAM, and 1TB of Apple SSD storage. The VM used by Docker was managed by VMWare Fusion +and had 4 CPUs and 2 GB of RAM. These number should be used for relative perfomance comparisons as the +results on another system may vary considerably. + +| Test | 1 Thread | 2 Threads | 4 Threads | 8 Threads | +|------------------------ |---------:|----------:|----------:|----------:| +|Flume Avro ||||| +|- Batch Size 1 - JSON |49.11 |46.54 |46.70 |44.92 | +|- Batch Size 1 - RFC5424 |48.30 |45.79 |46.31 |45.50 | +|- Batch Size 100 - JSON | 6.33 |3.87 |3.57 |3.84 | +|- Batch Size 100 - RFC5424 | 6.08 |3.69 |3.22 |3.11 | +|- Batch Size 1000 - JSON | 4.83 |3.20 |3.02 |2.11 | +|- Batch Size 1000 - RFC5424 | 4.70 |2.40 |2.37 |2.37 | +|Flume Embedded ||||| +| - RFC5424 |3.58 |2.10 |2.10 |2.70 | +| - JSON |4.20 |2.49 |3.53 |2.90 | +|Kafka Local JSON ||||| +| - sendSync true |58.46 |38.55 |19.59 |19.01 | +| - sendSync false |9.8 |10.8 |12.23 |11.36 | +|Console||||| +| - JSON / Kubernetes |3.03 |3.11 |3.04 |2.51 | +| - JSON |2.80 |2.74 |2.54 |2.35 | +| - Docker fluentd driver |10.65 |9.92 |10.42 |10.27 | +|Rolling File||||| +| - RFC5424 |1.65 |0.94 |1.22 |1.55 +| - JSON |1.90 |0.95 |1.57 |1.94 | +|TCP - Fluent Bit - JSON |2.34 |2.167 |1.67 |2.50 | +|Async Logger||||| +|- TCP - Fluent Bit - JSON|0.90 |0.58 |0.36 |0.48 | +|- Console - JSON |0.83 |0.57 |0.55 |0.61 | +|- Flume Avro - 1000 - JSON|0.76 |0.37 |0.45 |0.68 | + +Notes: + +1. Flume Avro - Buffering is controlled by the batch size. Each send is complete when the remote +acknowledges the batch was written to its channel. These number seem to indicate Flume Avro could +benefit from using a pool of RPCClients, at least for a batchSize of 1. +1. Flume Embedded - This is essentially asynchronous as it writes to an in-memory buffer. It is +unclear why the performance isn't closer to the AsyncLogger results. +1. Kafka was run in standalone mode on the same laptop as the application. See sendSync set to true +requires waiting for an ack from Kafka for each log event. +1. Console - System.out is redirected to a file by Docker. Testing shows that it would be much +slower if it was writing to the terminal screen. +1. Rolling File - Test uses the default buffer size of 8K. +1. TCP to Fluent Bit - The Socket Appender uses a default buffer size of 8K. +1. Async Loggers - These all write to a circular buffer and return to the application. The actual +I/O will take place on a separate thread. If writing the events is performed more slowly than +events are being created eventually the buffer will fill up and logging will be performed at +the same pace that log events are written. + +## Logging Recommendations + +1. Use asynchronous logging unless guaranteed delivery is absolutely required. As +the performance numbers show, so long as the volume of logging is not high enough to fill up the +circular buffer the overhead of logging will almost be unnoticeable to the application. +1. If overall performance is a consideration or you require multiline events such as stack traces +be processed properly then log via TCP to a companion container that acts as a log forwarder. Use the +Log4j Docker Lookup to add the container information to each log event. +1. Whenvever guaranteed delivery is required use Flume Avro with a batch size of 1 or another Appender such +as the Kafka Appender with syncSend set to true that only return control after the downstream agent +acknowledges receipt of the event. Beware that using an Appender that writes each event individually should +be kept to a minimum since it is much slower than sending buffered events. +1. Logging to files within the container is discouraged. Doing so reuires that a volume be declared in +the Docker configuration and that the file be tailed by a log forwarder. However, it performs +better than logging to the standard output stream. If logging via TCP is not an option and +proper multiline handling is required then consider this option. \ No newline at end of file diff --git a/src/site/resources/images/DockerFluentd.drawio b/src/site/resources/images/DockerFluentd.drawio new file mode 100644 index 0000000..8664a4e --- /dev/null +++ b/src/site/resources/images/DockerFluentd.drawio @@ -0,0 +1 @@ +<mxfile modified="2019-03-31T19:39:23.743Z" host="www.draw.io" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36" etag="ga3EPmrR2nb5J0BswPKH" version="10.5.9" type="device"><diagram id="5SG5qToZW8ppSJovxSJi" name="Page-1">3VjZctsgFP0aP6ajzbL1mHhppk2mSZ1pk6cOEdiiwUJFeMvX92KBdsdprHQ69YMNB7iIc+4R4J47Wm4/CpRE1xwT1nMsvO25457jOJY7hB+F7DLEtmwrQxaCYo0VwIw+E9NRoyuKSVrpKDlnkiZVMORxTEJZwZAQfFPtNuesOmuCFqQBzELEmuh3imWkUduy [...] \ No newline at end of file diff --git a/src/site/resources/images/DockerFluentd.png b/src/site/resources/images/DockerFluentd.png new file mode 100644 index 0000000..edcd350 Binary files /dev/null and b/src/site/resources/images/DockerFluentd.png differ diff --git a/src/site/resources/images/DockerFluentdAggregator.drawio b/src/site/resources/images/DockerFluentdAggregator.drawio new file mode 100644 index 0000000..4b312ff --- /dev/null +++ b/src/site/resources/images/DockerFluentdAggregator.drawio @@ -0,0 +1 @@ +<mxfile modified="2019-03-31T20:13:53.215Z" host="www.draw.io" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36" etag="M96szTU_Fw0FbEZ1bxDf" version="10.5.9" type="device"><diagram id="5SG5qToZW8ppSJovxSJi" name="Page-1">zVfLctowFP0alun4hcFLYqCZNpkmpdMmq45iC1uNsFxZgMnX9wpLfhMyAWbKIpGOpCv7nHt05YHtr/LPHKXxHQsxHVhGmA/s6cCyLMMewz+J7ArENEyjQCJOQoVVwIK8Yj1RoWsS4qwxUTBGBUmbYMCSBAeigSHO2bY5bcloc9cURbgDLAJEu+gvEopYoaZh [...] \ No newline at end of file diff --git a/src/site/resources/images/DockerFluentdAggregator.png b/src/site/resources/images/DockerFluentdAggregator.png new file mode 100644 index 0000000..846c5a6 Binary files /dev/null and b/src/site/resources/images/DockerFluentdAggregator.png differ diff --git a/src/site/resources/images/DockerLogFile.drawio b/src/site/resources/images/DockerLogFile.drawio new file mode 100644 index 0000000..1de4dd3 --- /dev/null +++ b/src/site/resources/images/DockerLogFile.drawio @@ -0,0 +1 @@ +<mxfile modified="2019-03-31T20:17:04.511Z" host="www.draw.io" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36" etag="CK1puqeNANjE1A8R_XEg" version="10.5.9" type="device"><diagram id="5SG5qToZW8ppSJovxSJi" name="Page-1">3VhNc5swEP01PrYjwNjm6BDTr3ia1jNJfFSQbNQKRIUIkF9fyQhjkB3HiZOZNodEu1qt0Hu7D5GB48flJw7TaM4QpgMboHLgXA5s2wbORP5Rnqr2WMACtWfNCdK+1rEgj7gJ1N6cIJx1AgVjVJC06wxZkuBQdHyQc1Z0w1aMdndN4RobjkUIqem9JUhE2msB [...] \ No newline at end of file diff --git a/src/site/resources/images/DockerLogFile.png b/src/site/resources/images/DockerLogFile.png new file mode 100644 index 0000000..5ec366e Binary files /dev/null and b/src/site/resources/images/DockerLogFile.png differ diff --git a/src/site/resources/images/DockerStdout.drawio b/src/site/resources/images/DockerStdout.drawio new file mode 100644 index 0000000..9b35764 --- /dev/null +++ b/src/site/resources/images/DockerStdout.drawio @@ -0,0 +1 @@ +<mxfile modified="2019-03-31T20:22:29.524Z" host="www.draw.io" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36" etag="tZjZN1zqtIJ6xoe4SOn7" version="10.5.9" type="device"><diagram id="5SG5qToZW8ppSJovxSJi" name="Page-1">3VjZctowFP0aHtPxAgY/Jga6DGnTMtOER2ELW42wXFlm6ddXwvIqOxAwSac8gHR1tfice48u7pnOeveRgii4Jx7EPUPzdj1z3DMMQzNH/EdY9qlF13QttfgUedJWGOboD8wcpTVBHowrjowQzFBUNbokDKHLKjZAKdlW3VYEV3eNgA8Vw9wFWLU+Io8F0qpr [...] \ No newline at end of file diff --git a/src/site/resources/images/DockerStdout.png b/src/site/resources/images/DockerStdout.png new file mode 100644 index 0000000..546e896 Binary files /dev/null and b/src/site/resources/images/DockerStdout.png differ diff --git a/src/site/resources/images/DockerTCP.drawio b/src/site/resources/images/DockerTCP.drawio new file mode 100644 index 0000000..733bbf6 --- /dev/null +++ b/src/site/resources/images/DockerTCP.drawio @@ -0,0 +1 @@ +<mxfile modified="2019-03-31T20:25:12.743Z" host="www.draw.io" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36" etag="W_kcBCjDz_BLlawZ2mwf" version="10.5.9" type="device"><diagram id="5SG5qToZW8ppSJovxSJi" name="Page-1">3VdNc5swEP01PqYDKOBwTLBpppPMpHWnTU4dBQlQIxARcsD99V0ZYT7zMXXaQ32wtW9XK/TeapEXKMjqjxIX6bUglC8ci9QLtFo4jmOhM/jRyK5BbMu2GiSRjBisAzbsF20DDbplhJaDQCUEV6wYgpHIcxqpAYalFNUwLBZ8uGqBEzoBNhHmU/Q7Iyo1qG1Z [...] \ No newline at end of file diff --git a/src/site/resources/images/DockerTCP.png b/src/site/resources/images/DockerTCP.png new file mode 100644 index 0000000..4a30200 Binary files /dev/null and b/src/site/resources/images/DockerTCP.png differ diff --git a/src/site/resources/images/LoggerAggregator.drawio b/src/site/resources/images/LoggerAggregator.drawio new file mode 100644 index 0000000..91c5668 --- /dev/null +++ b/src/site/resources/images/LoggerAggregator.drawio @@ -0,0 +1 @@ +<mxfile modified="2019-03-31T20:10:25.947Z" host="www.draw.io" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36" etag="_GAVgQ1D8SCqD5mIJyzn" version="10.5.9" type="device"><diagram id="5SG5qToZW8ppSJovxSJi" name="Page-1">tVbbcpswEP0aHtvhYtPw6Gs7nWTGrWfa5FFBC6gRiAph4359V0bcnUsnLg+29uxqJZ2zWrC8VVp9liRP7gQFbrk2rSxvbbmua3s3+KeRU404tmPXSCwZNVgH7NkfaAINWjIKxSBQCcEVy4dgKLIMQjXAiJTiOAyLBB+umpMYJsA+JHyK/mRUJQZ1bLtzfAEW [...] \ No newline at end of file diff --git a/src/site/resources/images/LoggerAggregator.png b/src/site/resources/images/LoggerAggregator.png new file mode 100644 index 0000000..78ebf0a Binary files /dev/null and b/src/site/resources/images/LoggerAggregator.png differ diff --git a/src/site/site.xml b/src/site/site.xml index 1a49332..7fe1c72 100644 --- a/src/site/site.xml +++ b/src/site/site.xml @@ -111,6 +111,7 @@ <item name="Usage" href="/manual/usage.html" collapse="true"> <item name="Static vs non-Static Loggers" href="/manual/usage.html#StaticVsNonStatic"/> <item name="Logger Name vs Class Name" href="/manual/usage.html#LoggerVsClass"/> + <item name="Logging in the Cloud" href="/manual/cloud.html"/> </item> <item name="Web Applications and JSPs" href="/manual/webapp.html" collapse="true"> @@ -125,6 +126,7 @@ <item name="Lookups" href="/manual/lookups.html" collapse="true"> <item name="Context Map" href="/manual/lookups.html#ContextMapLookup"/> <item name="Date" href="/manual/lookups.html#DateLookup"/> + <item name="Docker" href="manual/lookups.html#DockerLookup"/> <item name="Environment" href="/manual/lookups.html#EnvironmentLookup"/> <item name="Java" href="/manual/lookups.html#JavaLookup"/> <item name="JNDI" href="/manual/lookups.html#JndiLookup"/> @@ -295,6 +297,8 @@ <item name="Log4j Cassandra appender" href="log4j-cassandra/index.html"/> <item name="Log4j IO Streams" href="log4j-iostreams/index.html"/> <item name="Log4j Liquibase Binding" href="log4j-liquibase/index.html"/> + <item name="Log4j Docker Support" href="log4j-docker/index.html"/> + <item name="Log4j Spring Cloud Config Client" href="log4j-spring-cloud-config/log4j-spring-cloud-config-client/index.html"/> </menu> <menu name="Project Information" img="icon-info-sign"> diff --git a/src/site/xdoc/manual/configuration.xml.vm b/src/site/xdoc/manual/configuration.xml.vm index 119c60e..e0291d7 100644 --- a/src/site/xdoc/manual/configuration.xml.vm +++ b/src/site/xdoc/manual/configuration.xml.vm @@ -2386,6 +2386,84 @@ public class AwesomeTest { <td>Disables message pattern lookups globally when set to <tt>true</tt>. This is equivalent to defining all message patterns using <tt>%m{nolookups}</tt>.</td> </tr> + <tr> + <td><a name="log4j2.trustStore.location "/>log4j2.trustStore.location</td> + <td>LOG4J_TRUST_STORE_LOCATION</td> + <td></td> + <td>The location of the trust store. If not provided the default trust store will be used.</td> + </tr> + <tr> + <td><a name="log4j2.trustStore.password"/>log4j2.trustStore.password</td> + <td>LOG4J_TRUST_STORE_PASSWORD</td> + <td></td> + <td>Password needed to access the trust store.</td> + </tr> + <tr> + <td><a name="log4j2.trustStore.passwordFile"/>log4j2.trustStore.passwordFile</td> + <td>LOG4J_TRUST_STORE_PASSWORD_FILE</td> + <td></td> + <td>The location of a file that contains the password for the trust store.</td> + </tr> + <tr> + <td><a name="log4j2.trustStore.passwordEnvironmentVariable"/>log4j2.trustStore.passwordEnvironmentVariable</td> + <td>LOG4J_TRUST_STORE_PASSWORD_ENVIRONMENT_VARIABLE</td> + <td></td> + <td>The name of the environment variable that contains the trust store password.</td> + </tr> + <tr> + <td><a name="log4j2.trustStore.type"/>log4j2.trustStore.type</td> + <td>LOG4J_TRUST_STORE_TYPE</td> + <td></td> + <td>The type of key store used for the trust store.</td> + </tr> + <tr> + <td><a name="log4j2.trustStore.keyManagerFactoryAlgorithm"/>log4j2.trustStore.keyManagerFactoryAlgorithm</td> + <td>LOG4J_TRUST_STORE_KEY_MANAGER_FACTORY_ALGORITHM</td> + <td></td> + <td>Java cryptographic algorithm.</td> + </tr> + <tr> + <td><a name="log4j2.keyStore.location "/>log4j2.keyStore.location </td> + <td>LOG4J_KEY_STORE_LOCATION</td> + <td></td> + <td>The location of the key store. If not provided the default key store will be used.</td> + </tr> + <tr> + <td><a name="log4j2.keyStore.password"/>log4j2.keyStore.password</td> + <td>LOG4J_KEY_STORE_PASSWORD</td> + <td></td> + <td>Password needed to access the key store.</td> + </tr> + <tr> + <td><a name="log4j2.keyStore.passwordFile"/>log4j2.keyStore.passwordFile</td> + <td>LOG4J_KEY_STORE_PASSWORD_FILE</td> + <td></td> + <td>The location of a file that contains the password for the key store.</td> + </tr> + <tr> + <td><a name="log4j2.keyStore.passwordEnvironmentVariable"/>log4j2.keyStore.passwordEnvironmentVariable</td> + <td>LOG4J_KEY_STORE_PASSWORD_ENVIRONMENT_VARIABLE</td> + <td></td> + <td>The name of the environment variable that contains the key store password.</td> + </tr> + <tr> + <td><a name="log4j2.keyStore.type"/>log4j2.keyStore.type</td> + <td>LOG4J_KEY_STORE_TYPE</td> + <td></td> + <td>The type of key store.</td> + </tr> + <tr> + <td><a name="log4j2.keyStore.keyManagerFactoryAlgorithm"/>log4j2.keyStore.keyManagerFactoryAlgorithm</td> + <td>LOG4J_KEY_STORE_KEY_MANAGER_FACTORY_ALGORITHM</td> + <td></td> + <td>Java cryptographic algorithm.</td> + </tr> + <tr> + <td><a name="log4j2.ssl.verifyHostName"/>log4j2.ssl.verifyHostName</td> + <td></td> + <td>false</td> + <td>true or false if the host name should be verified</td> + </tr> </table> </subsection> diff --git a/src/site/xdoc/manual/lookups.xml b/src/site/xdoc/manual/lookups.xml index 255054a..051033f 100644 --- a/src/site/xdoc/manual/lookups.xml +++ b/src/site/xdoc/manual/lookups.xml @@ -67,6 +67,31 @@ <SizeBasedTriggeringPolicy size="500" /> </RollingFile>]]></pre> </subsection> + <a name="DockerLookup"/> + <subsection name="Docker Lookup"> + <p> + The DockerLookup can be used to lookup attributes from the Docker container the application is running + in. + </p> + Log4j Docker provides access to the following container attributes: + <table> + <tr><td>containerId</td><td>The full id assigned to the container.</td></tr> + <tr><td>containreName</td><td>The name assigned to the container.</td></tr> + <tr><td>imageId</td><td>The id assigned to the image.</td></tr> + <tr><td>imageName</td><td>The name assigned to the image.</td></tr> + <tr><td>shortContainerId</td><td>The first 12 characters of the container id.</td></tr> + <tr><td>shortImageId</td><td>The first 12 characters of the image id.</td></tr> + </table> + <pre class="prettyprint linenums"><![CDATA[ +<JsonLayout properties="true" compact="true" eventEol="true"> + <KeyValuePair key="containerId" value="${docker:containerId}"/> + <KeyValuePair key="containerName" value="${docker:containerName}"/> + <KeyValuePair key="imageName" value="${docker:imageName}"/> +</JsonLayout>]]></pre> + <p> + This Lookup is subject to the requirements listed at <a href="../log4j-docker/index.html">Log4j Docker Support</a> + </p> + </subsection> <a name="EnvironmentLookup"/> <subsection name="Environment Lookup"> <p> diff --git a/src/site/xdoc/runtime-dependencies.xml b/src/site/xdoc/runtime-dependencies.xml index 9429feb..4a5c118 100644 --- a/src/site/xdoc/runtime-dependencies.xml +++ b/src/site/xdoc/runtime-dependencies.xml @@ -81,20 +81,36 @@ </td> </tr> <tr> + <td>log4j-appserver</td> + <td>org.apache.logging.log4j.appserver</td> + <td>Automatic Module</td> + </tr> + <tr> + <td>log4j-cassandra</td> + <td>org.apache.logging.log4j.cassandra</td> + <td>Automatic Module</td> + </tr> + <tr> <td>log4j-core</td> <td>org.apache.logging.log4j.core</td> <td>Automatic Module</td> </tr> <tr> - <td>log4j-1.2-api</td> - <td>org.apache.log4j</td> + <td>log4j-couchdb</td> + <td>org.apache.logging.log4j.couchdb</td> <td>Automatic Module</td> </tr> <tr> - <td>log4j-appserver</td> - <td>org.apache.logging.log4j.appserver</td> + <td>log4j-docker</td> + <td>org.apache.logging.log4j.docker</td> + <td>Automatic Module</td> + </tr> + <tr> + <td>log4j-1.2-api</td> + <td>org.apache.log4j</td> <td>Automatic Module</td> </tr> + <tr> <td>log4j-flume-ng</td> <td>org.apache.logging.log4j.flume</td> @@ -121,39 +137,30 @@ <td>Automatic Module</td> </tr> <tr> - <td>log4j-couchdb</td> - <td>org.apache.logging.log4j.couchdb</td> - <td>Automatic Module</td> - </tr> - <tr> <td>log4j-mongodb</td> <td>org.apache.logging.log4j.mongodb</td> <td>Automatic Module</td> </tr> - <tr> - <td>log4j-cassandra</td> - <td>org.apache.logging.log4j.cassandra</td> - <td>Automatic Module</td> - </tr> + <tr> <td>log4j-osgi</td> <td>org.apache.logging.log4j.osgi</td> <td>Automatic Module. Unclear how OSGi will support Java modules.</td> </tr> <tr> - <td>log4j-slf4j-impl</td> - <td>org.apache.logging.log4j.slf4j.impl</td> - <td>Automatic Module. May require renaming should SLF4J ever require all implementations to have the same - module name.</td> - </tr> + <td>log4j-slf4j-impl</td> + <td>org.apache.logging.log4j.slf4j.impl</td> + <td>Automatic Module. May require renaming should SLF4J ever require all implementations to have the same + module name.</td> + </tr> <tr> - <td>log4j-to-slf4j</td> - <td>org.apache.logging.log4j.slf4j</td> + <td>log4j-taglib</td> + <td>org.apache.logging.log4j.taglib</td> <td>Automatic Module</td> </tr> <tr> - <td>log4j-taglib</td> - <td>org.apache.logging.log4j.taglib</td> + <td>log4j-to-slf4j</td> + <td>org.apache.logging.log4j.slf4j</td> <td>Automatic Module</td> </tr> <tr> @@ -286,7 +293,14 @@ </td> </tr> </table> - + <a name="log4j-docker"/> + <h4>log4j-docker</h4> + <p> + <a href="log4j-docker/index.html">Log4j Docker Support</a> requires + <a href="https://github.com/FasterXML/jackson";>Jackson annotations, core, and databind</a>. See the + <a href="log4j-docker/dependencies.html#Dependency_Tree">Dependency Tree</a> for the exact list of + JAR files needed. + </p> <a name="log4j-jcl" /> <h4>log4j-jcl</h4> <p> @@ -348,7 +362,17 @@ The persistent agent uses Berkeley DB. See the <a href="log4j-flume-ng/dependencies.html#Dependency_Tree">Dependency Tree</a> for the exact list of JAR files needed. </p> - + <a name="log4j-spring-cloud-config-client"/> + <h4>log4j-spring-cloud-config-client</h4> + <p> + <a href="log4j-spring-cloud-config/log4j-spring-cloud-config-client/index.html">Log4j Spring Cloud Config Client</a> requires + <a href="https://spring.io/projects/spring-cloud-config";>Spring Cloud Config</a>. + <a href="https://spring.io/projects/spring-cloud-bus";>Spring Cloud Bus</a> is required if notification of logging + configuration changes is desired. <a href="https://spring.io/projects/spring-boot";>Spring Boot</a> is required + but applications do not have to be packaged as a Spring Boot application. + <a href="log4j-spring-cloud-config/log4j-spring-cloud-config-client/dependencies.html#Dependency_Tree">Dependency Tree</a> for the exact list of + JAR files needed. + </p> <a name="log4j-taglib" /> <h4>log4j-taglib</h4> <p>
