LOG4J2-2709 - Allow message portion of GELF layout to be formatted using a PatternLayout. Allow ThreadContext attributes to be explicitly included or excluded in the GelfLayout.
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractStringLayout.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractStringLayout.java
index 759fe38..76c00b2 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractStringLayout.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/AbstractStringLayout.java
@@ -93,13 +93,19 @@
default boolean requiresLocation() {
return false;
}
+
+ default StringBuilder toSerializable(final LogEvent event, final StringBuilder builder) {
+ builder.append(toSerializable(event));
+ return builder;
+ }
}
/**
* Variation of {@link Serializer} that avoids allocating temporary objects.
+ * As of 2.13 this interface was merged into the Serializer interface.
* @since 2.6
*/
- public interface Serializer2 {
+ public interface Serializer2 {
StringBuilder toSerializable(final LogEvent event, final StringBuilder builder);
}
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/GelfLayout.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/GelfLayout.java
index b0c28fb..268ec6a 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/GelfLayout.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/GelfLayout.java
@@ -20,11 +20,15 @@
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.layout.internal.ExcludeChecker;
+import org.apache.logging.log4j.core.layout.internal.IncludeChecker;
+import org.apache.logging.log4j.core.layout.internal.ListChecker;
import org.apache.logging.log4j.core.lookup.StrSubstitutor;
import org.apache.logging.log4j.core.net.Severity;
import org.apache.logging.log4j.core.util.JsonUtils;
import org.apache.logging.log4j.core.util.KeyValuePair;
import org.apache.logging.log4j.core.util.NetUtils;
+import org.apache.logging.log4j.core.util.Patterns;
import org.apache.logging.log4j.message.Message;
import org.apache.logging.log4j.plugins.Node;
import org.apache.logging.log4j.plugins.Plugin;
@@ -42,7 +46,9 @@
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.GZIPOutputStream;
@@ -97,6 +103,8 @@
private final boolean includeStacktrace;
private final boolean includeThreadContext;
private final boolean includeNullDelimiter;
+ private final PatternLayout layout;
+ private final FieldWriter fieldWriter;
public static class Builder<B extends Builder<B>> extends AbstractStringLayout.Builder<B>
implements org.apache.logging.log4j.plugins.util.Builder<GelfLayout> {
@@ -122,6 +130,15 @@
@PluginBuilderAttribute
private boolean includeNullDelimiter = false;
+ @PluginBuilderAttribute
+ private String threadContextIncludes = null;
+
+ @PluginBuilderAttribute
+ private String threadContextExcludes = null;
+
+ @PluginBuilderAttribute
+ private String messagePattern = null;
+
public Builder() {
super();
setCharset(StandardCharsets.UTF_8);
@@ -129,8 +146,39 @@
@Override
public GelfLayout build() {
+ ListChecker checker = null;
+ if (threadContextExcludes != null) {
+ final String[] array = threadContextExcludes.split(Patterns.COMMA_SEPARATOR);
+ if (array.length > 0) {
+ List<String> excludes = new ArrayList<>(array.length);
+ for (final String str : array) {
+ excludes.add(str.trim());
+ }
+ checker = new ExcludeChecker(excludes);
+ }
+ }
+ if (threadContextIncludes != null) {
+ final String[] array = threadContextIncludes.split(Patterns.COMMA_SEPARATOR);
+ if (array.length > 0) {
+ List<String> includes = new ArrayList<>(array.length);
+ for (final String str : array) {
+ includes.add(str.trim());
+ }
+ checker = new IncludeChecker(includes);
+ }
+ }
+ if (checker == null) {
+ checker = ListChecker.NOOP_CHECKER;
+ }
+ PatternLayout patternLayout = null;
+ if (messagePattern != null) {
+ patternLayout = PatternLayout.newBuilder().setPattern(messagePattern)
+ .setAlwaysWriteExceptions(includeStacktrace)
+ .setConfiguration(getConfiguration())
+ .build();
+ }
return new GelfLayout(getConfiguration(), host, additionalFields, compressionType, compressionThreshold,
- includeStacktrace, includeThreadContext, includeNullDelimiter);
+ includeStacktrace, includeThreadContext, includeNullDelimiter, checker, patternLayout);
}
public String getHost() {
@@ -230,10 +278,42 @@
this.additionalFields = additionalFields;
return asBuilder();
}
+
+ /**
+ * The pattern to use to format the message.
+ * @param pattern the pattern string.
+ * @return this builder
+ */
+ public B setMessagePattern(final String pattern) {
+ this.messagePattern = pattern;
+ return asBuilder();
+ }
+
+ /**
+ * A comma separated list of thread context keys to include;
+ * @param mdcIncludes the list of keys.
+ * @return this builder
+ */
+ public B setMdcIncludes(final String mdcIncludes) {
+ this.threadContextIncludes = mdcIncludes;
+ return asBuilder();
+ }
+
+ /**
+ * A comma separated list of thread context keys to include;
+ * @param mdcExcludes the list of keys.
+ * @return this builder
+ */
+ public B setMdcExcludes(final String mdcExcludes) {
+ this.threadContextExcludes = mdcExcludes;
+ return asBuilder();
+ }
}
- private GelfLayout(final Configuration config, final String host, final KeyValuePair[] additionalFields, final CompressionType compressionType,
- final int compressionThreshold, final boolean includeStacktrace, final boolean includeThreadContext, final boolean includeNullDelimiter) {
+ private GelfLayout(final Configuration config, final String host, final KeyValuePair[] additionalFields,
+ final CompressionType compressionType, final int compressionThreshold, final boolean includeStacktrace,
+ final boolean includeThreadContext, final boolean includeNullDelimiter, final ListChecker listChecker,
+ final PatternLayout patternLayout) {
super(config, StandardCharsets.UTF_8, null, null);
this.host = host != null ? host : NetUtils.getLocalHostname();
this.additionalFields = additionalFields != null ? additionalFields : new KeyValuePair[0];
@@ -252,6 +332,27 @@
if (includeNullDelimiter && compressionType != CompressionType.OFF) {
throw new IllegalArgumentException("null delimiter cannot be used with compression");
}
+ this.fieldWriter = new FieldWriter(listChecker);
+ this.layout = patternLayout;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("host=").append(host);
+ sb.append(", compressionType=").append(compressionType.toString());
+ sb.append(", compressionThreshold=").append(compressionThreshold);
+ sb.append(", includeStackTrace=").append(includeStacktrace);
+ sb.append(", includeThreadContext=").append(includeThreadContext);
+ sb.append(", includeNullDelimiter=").append(includeNullDelimiter);
+ String threadVars = fieldWriter.getChecker().toString();
+ if (threadVars.length() > 0) {
+ sb.append(", ").append(threadVars);
+ }
+ if (layout != null) {
+ sb.append(", PatternLayout{").append(layout.toString()).append("}");
+ }
+ return sb.toString();
}
@PluginFactory
@@ -342,14 +443,21 @@
}
}
if (includeThreadContext) {
- event.getContextData().forEach(WRITE_KEY_VALUES_INTO, builder);
+ event.getContextData().forEach(fieldWriter, builder);
}
- if (event.getThrown() != null) {
+
+ if (event.getThrown() != null || layout != null) {
builder.append("\"full_message\":\"");
- if (includeStacktrace) {
- JsonUtils.quoteAsString(formatThrowable(event.getThrown()), builder);
+ if (layout != null) {
+ final StringBuilder messageBuffer = getMessageStringBuilder();
+ layout.serialize(event, messageBuffer);
+ JsonUtils.quoteAsString(messageBuffer, builder);
} else {
- JsonUtils.quoteAsString(event.getThrown().toString(), builder);
+ if (includeStacktrace) {
+ JsonUtils.quoteAsString(formatThrowable(event.getThrown()), builder);
+ } else {
+ JsonUtils.quoteAsString(event.getThrown().toString(), builder);
+ }
}
builder.append(QC);
}
@@ -357,7 +465,7 @@
builder.append("\"short_message\":\"");
final Message message = event.getMessage();
if (message instanceof CharSequence) {
- JsonUtils.quoteAsString(((CharSequence)message), builder);
+ JsonUtils.quoteAsString(((CharSequence) message), builder);
} else if (gcFree && message instanceof StringBuilderFormattable) {
final StringBuilder messageBuffer = getMessageStringBuilder();
try {
@@ -381,14 +489,26 @@
return value != null && value.contains("${");
}
- private static final TriConsumer<String, Object, StringBuilder> WRITE_KEY_VALUES_INTO = new TriConsumer<String, Object, StringBuilder>() {
+ private static class FieldWriter implements TriConsumer<String, Object, StringBuilder> {
+ private final ListChecker checker;
+
+ FieldWriter(ListChecker checker) {
+ this.checker = checker;
+ }
+
@Override
public void accept(final String key, final Object value, final StringBuilder stringBuilder) {
- stringBuilder.append(QU);
- JsonUtils.quoteAsString(key, stringBuilder);
- stringBuilder.append("\":\"");
- JsonUtils.quoteAsString(toNullSafeString(String.valueOf(value)), stringBuilder);
- stringBuilder.append(QC);
+ if (checker.check(key)) {
+ stringBuilder.append(QU);
+ JsonUtils.quoteAsString(key, stringBuilder);
+ stringBuilder.append("\":\"");
+ JsonUtils.quoteAsString(toNullSafeString(String.valueOf(value)), stringBuilder);
+ stringBuilder.append(QC);
+ }
+ }
+
+ public ListChecker getChecker() {
+ return checker;
}
};
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/PatternLayout.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/PatternLayout.java
index 064b1ef..10a36c5 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/PatternLayout.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/PatternLayout.java
@@ -184,6 +184,10 @@
return eventSerializer.toSerializable(event);
}
+ public void serialize(final LogEvent event, StringBuilder stringBuilder) {
+ eventSerializer.toSerializable(event, stringBuilder);
+ }
+
@Override
public void encode(final LogEvent event, final ByteBufferDestination destination) {
if (!(eventSerializer instanceof Serializer2)) {
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/Rfc5424Layout.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/Rfc5424Layout.java
index 9584c18..969a769 100644
--- a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/Rfc5424Layout.java
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/Rfc5424Layout.java
@@ -35,6 +35,9 @@
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.appender.TlsSyslogFrame;
import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.layout.internal.ExcludeChecker;
+import org.apache.logging.log4j.core.layout.internal.IncludeChecker;
+import org.apache.logging.log4j.core.layout.internal.ListChecker;
import org.apache.logging.log4j.plugins.Node;
import org.apache.logging.log4j.plugins.Plugin;
import org.apache.logging.log4j.plugins.PluginAttribute;
@@ -112,7 +115,6 @@
private final List<String> mdcIncludes;
private final List<String> mdcRequired;
private final ListChecker listChecker;
- private final ListChecker noopChecker = new NoopChecker();
private final boolean includeNewLine;
private final String escapeNewLine;
private final boolean useTlsMessageFormat;
@@ -146,12 +148,12 @@
this.messageId = messageId;
this.useTlsMessageFormat = useTLSMessageFormat;
this.localHostName = NetUtils.getLocalHostname();
- ListChecker c = null;
+ ListChecker checker = null;
if (excludes != null) {
final String[] array = excludes.split(Patterns.COMMA_SEPARATOR);
if (array.length > 0) {
- c = new ExcludeChecker();
mdcExcludes = new ArrayList<>(array.length);
+ checker = new ExcludeChecker(mdcExcludes);
for (final String str : array) {
mdcExcludes.add(str.trim());
}
@@ -164,8 +166,8 @@
if (includes != null) {
final String[] array = includes.split(Patterns.COMMA_SEPARATOR);
if (array.length > 0) {
- c = new IncludeChecker();
mdcIncludes = new ArrayList<>(array.length);
+ checker = new IncludeChecker(mdcIncludes);
for (final String str : array) {
mdcIncludes.add(str.trim());
}
@@ -189,7 +191,7 @@
} else {
mdcRequired = null;
}
- this.listChecker = c != null ? c : noopChecker;
+ this.listChecker = checker != null ? checker : ListChecker.NOOP_CHECKER;
final String name = config == null ? null : config.getName();
configName = Strings.isNotEmpty(name) ? name : null;
this.fieldFormatters = createFieldFormatters(loggerFields, config);
@@ -513,7 +515,7 @@
sb.append('[');
sb.append(id);
if (!mdcSdId.toString().equals(id)) {
- appendMap(data.getPrefix(), data.getFields(), sb, noopChecker);
+ appendMap(data.getPrefix(), data.getFields(), sb, ListChecker.NOOP_CHECKER);
} else {
appendMap(data.getPrefix(), data.getFields(), sb, checker);
}
@@ -566,43 +568,6 @@
return PARAM_VALUE_ESCAPE_PATTERN.matcher(value).replaceAll("\\\\$0");
}
- /**
- * Interface used to check keys in a Map.
- */
- private interface ListChecker {
- boolean check(String key);
- }
-
- /**
- * Includes only the listed keys.
- */
- private class IncludeChecker implements ListChecker {
- @Override
- public boolean check(final String key) {
- return mdcIncludes.contains(key);
- }
- }
-
- /**
- * Excludes the listed keys.
- */
- private class ExcludeChecker implements ListChecker {
- @Override
- public boolean check(final String key) {
- return !mdcExcludes.contains(key);
- }
- }
-
- /**
- * Does nothing.
- */
- private class NoopChecker implements ListChecker {
- @Override
- public boolean check(final String key) {
- return true;
- }
- }
-
@Override
public String toString() {
final StringBuilder sb = new StringBuilder();
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/internal/ExcludeChecker.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/internal/ExcludeChecker.java
new file mode 100644
index 0000000..2841dd2
--- /dev/null
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/internal/ExcludeChecker.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+package org.apache.logging.log4j.core.layout.internal;
+
+import java.util.List;
+
+/**
+ * Excludes the listed keys.
+ */
+public class ExcludeChecker implements ListChecker {
+ private final List<String> list;
+
+ public ExcludeChecker(final List<String> list) {
+ this.list = list;
+ }
+
+ @Override
+ public boolean check(final String key) {
+ return !list.contains(key);
+ }
+
+ @Override
+ public String toString() {
+ return "ThreadContextExcludes=" + list.toString();
+ }
+}
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/internal/IncludeChecker.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/internal/IncludeChecker.java
new file mode 100644
index 0000000..c87b807
--- /dev/null
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/internal/IncludeChecker.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+package org.apache.logging.log4j.core.layout.internal;
+
+import java.util.List;
+
+/**
+ * Includes only the listed keys.
+ */
+public class IncludeChecker implements ListChecker {
+ private final List<String> list;
+
+ public IncludeChecker(final List<String> list) {
+ this.list = list;
+ }
+ @Override
+ public boolean check(final String key) {
+ return list.contains(key);
+ }
+
+ @Override
+ public String toString() {
+ return "ThreadContextIncludes=" + list.toString();
+ }
+}
+
diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/internal/ListChecker.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/internal/ListChecker.java
new file mode 100644
index 0000000..1993bc0
--- /dev/null
+++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/internal/ListChecker.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+package org.apache.logging.log4j.core.layout.internal;
+
+import java.util.List;
+
+/**
+ * Class Description goes here.
+ */
+
+public interface ListChecker {
+
+ static final NoopChecker NOOP_CHECKER = new NoopChecker();
+
+ boolean check(final String key);
+
+ /**
+ * Does nothing.
+ */
+ public class NoopChecker implements ListChecker {
+ @Override
+ public boolean check(final String key) {
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "";
+ }
+ }
+}
diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/GelfLayoutTest3.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/GelfLayoutTest3.java
new file mode 100644
index 0000000..9e02b65
--- /dev/null
+++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/GelfLayoutTest3.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+package org.apache.logging.log4j.core.layout;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.ThreadContext;
+import org.apache.logging.log4j.core.lookup.JavaLookup;
+import org.apache.logging.log4j.junit.LoggerContextRule;
+import org.junit.After;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class GelfLayoutTest3 {
+
+ @ClassRule
+ public static LoggerContextRule context = new LoggerContextRule("GelfLayoutTest3.xml");
+
+ @After
+ public void teardown() throws Exception {
+ ThreadContext.clearMap();
+ }
+
+ @Test
+ public void gelfLayout() throws IOException {
+ final Logger logger = context.getLogger();
+ ThreadContext.put("loginId", "rgoers");
+ ThreadContext.put("internalId", "12345");
+ logger.info("My Test Message");
+ final String gelf = context.getListAppender("list").getMessages().get(0);
+ final ObjectMapper mapper = new ObjectMapper();
+ final JsonNode json = mapper.readTree(gelf);
+ assertEquals("My Test Message", json.get("short_message").asText());
+ assertEquals("myhost", json.get("host").asText());
+ assertNotNull(json.get("_loginId"));
+ assertEquals("rgoers", json.get("_loginId").asText());
+ assertNull(json.get("_internalId"));
+ assertNull(json.get("_requestId"));
+ String message = json.get("full_message").asText();
+ assertTrue(message.contains("loginId=rgoers"));
+ assertTrue(message.contains("GelfLayoutTest3"));
+ }
+
+}
+
diff --git a/log4j-core/src/test/resources/GelfLayoutTest3.xml b/log4j-core/src/test/resources/GelfLayoutTest3.xml
new file mode 100644
index 0000000..fcb23d3
--- /dev/null
+++ b/log4j-core/src/test/resources/GelfLayoutTest3.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<Configuration status="WARN" name="GelfLayoutTest3">
+ <Appenders>
+ <List name="list">
+ <GelfLayout host="myhost" includeThreadContext="true" threadContextIncludes="requestId,loginId">
+ <MessagePattern>%d [%t] %-5p %X{requestId, loginId} %C{1.}.%M:%L - %m%n"</MessagePattern>
+ <KeyValuePair key="foo" value="FOO"/>
+ <KeyValuePair key="runtime" value="$${java:runtime}"/>
+ </GelfLayout>
+ </List>
+ </Appenders>
+
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="list"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/log4j-perf/src/main/java/org/apache/logging/log4j/perf/jmh/OutputBenchmark.java b/log4j-perf/src/main/java/org/apache/logging/log4j/perf/jmh/OutputBenchmark.java
new file mode 100644
index 0000000..3aa8a97
--- /dev/null
+++ b/log4j-perf/src/main/java/org/apache/logging/log4j/perf/jmh/OutputBenchmark.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+
+package org.apache.logging.log4j.perf.jmh;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.FileHandler;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Group;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Benchmarks Log4j 2, Log4j 1, Logback and JUL using the DEBUG level which is enabled for this test. The configuration
+ * for each uses a FileAppender
+ */
+// HOW TO RUN THIS TEST
+// java -jar log4j-perf/target/benchmarks.jar ".*OutputBenchmark.*" -f 1 -wi 10 -i 20
+//
+// RUNNING THIS TEST WITH 4 THREADS:
+// java -jar log4j-perf/target/benchmarks.jar ".*OutputBenchmark.*" -f 1 -wi 10 -i 20 -t 4
+@State(Scope.Thread)
+public class OutputBenchmark {
+ public static final String MESSAGE = "This is a debug message";
+
+ Logger log4j2Logger;
+
+
+ @State(Scope.Group)
+ public static class Redirect {
+ PrintStream defaultStream = System.out;
+
+ @Setup
+ public void setUp() throws Exception {
+ PrintStream ps = new PrintStream(new FileOutputStream("target/stdout.log"));
+ System.setOut(ps);
+ }
+
+ @TearDown
+ public void tearDown() {
+ PrintStream ps = System.out;
+ System.setOut(defaultStream);
+ ps.close();
+ }
+ }
+
+ @Setup
+ public void setUp() throws Exception {
+ System.setProperty("log4j.configurationFile", "log4j2-perf3.xml");
+
+ deleteLogFiles();
+
+ log4j2Logger = LogManager.getLogger(OutputBenchmark.class);
+ }
+
+ @TearDown
+ public void tearDown() {
+ System.clearProperty("log4j.configurationFile");
+
+ deleteLogFiles();
+ }
+
+ private void deleteLogFiles() {
+ final File outFile = new File("target/stdout.log");
+ final File log4j2File = new File ("target/testlog4j2.log");
+ log4j2File.delete();
+ outFile.delete();
+ }
+
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.SECONDS)
+ @Group("console")
+ @Benchmark
+ public void console() {
+ System.out.println(MESSAGE);
+ }
+
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.SECONDS)
+ @Group("file")
+ @Benchmark
+ public void log4j2File() {
+ log4j2Logger.debug(MESSAGE);
+ }
+
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.SECONDS)
+ @Group("redirect")
+ @Benchmark
+ public void redirect(Redirect redirect) {
+ System.out.println(MESSAGE);
+ }
+}
diff --git a/log4j-perf/src/main/resources/log4j2-perf3.xml b/log4j-perf/src/main/resources/log4j2-perf3.xml
new file mode 100644
index 0000000..1b977e7
--- /dev/null
+++ b/log4j-perf/src/main/resources/log4j2-perf3.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<Configuration name="XMLPerfTest" status="OFF">
+ <Appenders>
+ <File name="TestLogfile" fileName="target/testlog4j2.log" immediateFlush="false">
+ <PatternLayout>
+ <Pattern>%d %5p [%t] %c{1} %X{transactionId} - %m%n</Pattern>
+ </PatternLayout>
+ </File>
+ </Appenders>
+ <Loggers>
+ <Root level="debug">
+ <AppenderRef ref="TestLogfile"/>
+ </Root>
+ </Loggers>
+</Configuration>
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 09be11f..59dbd6b 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -34,9 +34,6 @@
<action issue="LOG4J2-2700" dev="mattsicker" type="add">
Add support for injecting plugin configuration via builder methods.
</action>
- <action issue="LOG4J2-2693" dev="mattsicker" type="fix">
- @PluginValue does not support attribute names besides "value".
- </action>
<action issue="LOG4J2-860" dev="mattsicker" type="update">
Unify plugin builders and plugin factories.
</action>
@@ -160,6 +157,13 @@
</action>
</release>
<release version="2.13.0" date="2019-MM-DD" description="GA Release 2.13.0">
+ <action issue="LOG4J2-2709" dev="rgoers" type="update">
+ Allow message portion of GELF layout to be formatted using a PatternLayout. Allow
+ ThreadContext attributes to be explicitly included or excluded in the GelfLayout.
+ </action>
+ <action issue="LOG4J2-2693" dev="mattsicker" type="fix">
+ @PluginValue does not support attribute names besides "value".
+ </action>
<action issue="LOG4J-2672" dev="rgoers" type="fix" due-to="Stephen Colebourne">
Add automatic module names where missing.
</action>
diff --git a/src/site/asciidoc/manual/layouts.adoc b/src/site/asciidoc/manual/layouts.adoc
index f5bcc41..888b5b5 100644
--- a/src/site/asciidoc/manual/layouts.adoc
+++ b/src/site/asciidoc/manual/layouts.adoc
@@ -225,13 +225,33 @@
|boolean
|Whether to include NULL byte as delimiter after each event (optional, default to false).
Useful for Graylog GELF TCP input. Cannot be used with compression.
+
+|messagePattern
+|String
+|The pattern to use to format the String. If not supplied only the text derived from the logging
+message will be used. See <<PatternLayout>> for information on the pattern
+strings
+
+|threadContextExcludes
+|String
+|A comma separated list of ThreadContext attributes to exclude when formatting the event. This
+attribute only applies when includeThreadContext="true" is specified. If threadContextIncludes
+are also specified this attribute will be ignored.
+
+|threadContextIncludes
+|String
+|A comma separated list of ThreadContext attributes to include when formatting the event. This
+attribute only applies when includeThreadContext="true" is specified. If threadContextExcludes
+are also specified this attribute will override them. ThreadContext fields specified here that
+have no value will be omitted.
|===
To include any custom field in the output, use following syntax:
[source,xml]
----
-<GelfLayout>
+<GelfLayout includeThreadContext="true" threadContextIncludes="loginId,requestId">
+ <MessagePattern>%d %5p [%t] %c{1} %X{loginId, requestId} - %m%n</MessagePattern>
<KeyValuePair key="additionalField1" value="constant value"/>
<KeyValuePair key="additionalField2" value="$${ctx:key}"/>
</GelfLayout>
diff --git a/src/site/markdown/manual/cloud.md b/src/site/markdown/manual/cloud.md
index 3663fb9..a9c919f 100644
--- a/src/site/markdown/manual/cloud.md
+++ b/src/site/markdown/manual/cloud.md
@@ -33,10 +33,19 @@
c. Log from Log4j directly to a logging forwarder or aggregator and bypass the docker logging driver.
1. When logging to stdout in Docker, log events pass through Java's standard output handling which is then directed
to the operating system so that the output can be piped into a file. The overhead of all this is measurably slower
-than just writing directly to a file as can be seen by the performance results below where logging
-to stdout is anywhere from 20 to 200% slower than logging directly to the file. However, these results alone
-would not be enough to argue against writing to the standard output stream as they only amount to about 20-30
-microseconds per logging call.
+than just writing directly to a file as can be seen in these benchmark results where logging
+to stdout is 16-20 times slower over repeated runs than logging directly to the file. The results below were obtained by
+running the [Output Benchmark](https://github.com/apache/logging-log4j2/blob/release-2.x/log4j-perf/src/main/java/org/apache/logging/log4j/perf/jmh/OutputBenchmark.java)
+on a 2018 MacBook Pro with a 2.9GHz Intel Core i9 processor and a 1TB SSD. However, these results alone would not be
+enough to argue against writing to the standard output stream as they only amount to about 14-25 microseconds
+per logging call vs 1.5 microseconds when writing to the file.
+ ```
+ Benchmark Mode Cnt Score Error Units
+ OutputBenchmark.console thrpt 20 39291.885 ± 3370.066 ops/s
+ OutputBenchmark.file thrpt 20 654584.309 ± 59399.092 ops/s
+ OutputBenchmark.redirect thrpt 20 70284.576 ± 7452.167 ops/s
+ ```
+
1. When performing audit logging using a framework such as log4j-audit guaranteed delivery of the audit events
is required. Many of the options for writing the output, including writing to the standard output stream, do
not guarantee delivery. In these cases the event must be delivered to a "forwarder" that acknowledges receipt
@@ -110,6 +119,69 @@
+## <a name="ELK"></a>Logging using ElasticSearch, Logstash, and Kibana
+
+The following configurations have been tested with an ELK stack and are known to work.
+
+### Log4j Configuration
+Use a socket appender with the GELF layout. Note that if the host name used by the socket appender has more than
+one ip address associated with its DNS entry the socket appender will fail through them all if needed.
+
+ <Socket name="Elastic" host="${sys:elastic.search.host}" port="12222" protocol="tcp" bufferedIo="true">
+ <GelfLayout includeStackTrace="true" host="${hostName}" includeThreadContext="true" includeNullDelimiter="true"
+ compressionType="OFF">
+ <ThreadContextIncludes>requestId,sessionId,loginId,userId,ipAddress,callingHost</ThreadContextIncludes>
+ <MessagePattern>%d [%t] %-5p %X{requestId, sessionId, loginId, userId, ipAddress} %C{1.}.%M:%L - %m%n</MessagePattern>
+ <KeyValuePair key="containerId" value="${docker:containerId:-}"/>
+ </GelfLayout>
+ </Socket>
+
+### Logstash Configuration
+
+ input {
+ gelf {
+ host => "localhost"
+ use_tcp => true
+ use_udp => false
+ port => 12222
+ type => "gelf"
+ }
+ }
+
+ filter {
+ # These are GELF/Syslog logging levels as defined in RFC 3164. Map the integer level to its human readable format.
+ translate {
+ field => "[level]"
+ destination => "[levelName]"
+ dictionary => {
+ "0" => "EMERG"
+ "1" => "ALERT"
+ "2" => "CRITICAL"
+ "3" => "ERROR"
+ "4" => "WARN"
+ "5" => "NOTICE"
+ "6" => "INFO"
+ "7" => "DEBUG"
+ }
+ }
+ }
+
+ output {
+ # (Un)comment for debugging purposes
+ # stdout { codec => rubydebug }
+ # Modify the hosts value to reflect where elasticsearch is installed.
+ elasticsearch {
+ hosts => ["http://localhost:9200/"]
+ index => "app-%{application}-%{+YYYY.MM.dd}"
+ }
+ }
+
+### Kibana
+With the above configurations the message field will contain a fully formatted log event just as it would appear in
+a file Appender. The ThreadContext attributes, custome fields, thread name, etc. will all be available as attributes
+on each log event that can be used for filtering.
+
+
## Managing Logging Configuration
Spring Boot provides another least common denominator approach to logging configuration. It will let you set the
@@ -211,7 +283,8 @@
the performance numbers show, so long as the volume of logging is not high enough to fill up the
circular buffer the overhead of logging will almost be unnoticeable to the application.
1. If overall performance is a consideration or you require multiline events such as stack traces
-be processed properly then log via TCP to a companion container that acts as a log forwarder. Use the
+be processed properly then log via TCP to a companion container that acts as a log forwarder or directly
+to a log aggregator as shown above in [Logging with ELK](#ELK). Use the
Log4j Docker Lookup to add the container information to each log event.
1. Whenever guaranteed delivery is required use Flume Avro with a batch size of 1 or another Appender such
as the Kafka Appender with syncSend set to true that only return control after the downstream agent
