APEXCORE-744 Add setting of predefined static logger appender properties
Added setting of the static logger appender properties: application name, container id, user name, service name, node name.
diff --git a/engine/src/main/java/com/datatorrent/stram/StreamingAppMaster.java b/engine/src/main/java/com/datatorrent/stram/StreamingAppMaster.java
index 7598b4f..de8ffa7 100644
--- a/engine/src/main/java/com/datatorrent/stram/StreamingAppMaster.java
+++ b/engine/src/main/java/com/datatorrent/stram/StreamingAppMaster.java
@@ -37,6 +37,7 @@
import org.apache.hadoop.yarn.util.Records;
import com.datatorrent.stram.debug.StdOutErrLog;
+import com.datatorrent.stram.util.LoggerUtil;
import com.datatorrent.stram.util.VersionInfo;
/**
@@ -56,6 +57,7 @@
*/
public static void main(final String[] args) throws Throwable
{
+ LoggerUtil.setupMDC("master");
StdOutErrLog.tieSystemOutAndErrToLog();
LOG.info("Master starting with classpath: {}", System.getProperty("java.class.path"));
diff --git a/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java b/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java
index 787f20b..152e6f1 100644
--- a/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java
+++ b/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java
@@ -4129,7 +4129,7 @@
public static void main(final String[] args) throws Exception
{
- LoggerUtil.addAppenders();
+ LoggerUtil.setupMDC("client");
final ApexCli shell = new ApexCli();
shell.preImpersonationInit(args);
String hadoopUserName = System.getenv("HADOOP_USER_NAME");
diff --git a/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java b/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java
index dd215a9..f5aaf35 100644
--- a/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java
+++ b/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java
@@ -286,6 +286,7 @@
*/
public static void main(String[] args) throws Throwable
{
+ LoggerUtil.setupMDC("worker");
StdOutErrLog.tieSystemOutAndErrToLog();
logger.debug("PID: " + System.getenv().get("JVM_PID"));
logger.info("Child starting with classpath: {}", System.getProperty("java.class.path"));
diff --git a/engine/src/main/java/com/datatorrent/stram/util/LoggerUtil.java b/engine/src/main/java/com/datatorrent/stram/util/LoggerUtil.java
index 14662e1..c862634 100644
--- a/engine/src/main/java/com/datatorrent/stram/util/LoggerUtil.java
+++ b/engine/src/main/java/com/datatorrent/stram/util/LoggerUtil.java
@@ -35,12 +35,17 @@
import org.apache.apex.log.LogFileInformation;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.Appender;
import org.apache.log4j.Category;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import org.apache.log4j.MDC;
import org.apache.log4j.PropertyConfigurator;
import org.apache.log4j.spi.DefaultRepositorySelector;
import org.apache.log4j.spi.HierarchyEventListener;
@@ -52,6 +57,9 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
+import com.datatorrent.stram.client.StramClientUtils;
+
+import static com.datatorrent.api.Context.DAGContext.APPLICATION_NAME;
import static com.datatorrent.api.Context.DAGContext.LOGGER_APPENDER;
/**
@@ -464,4 +472,33 @@
}
return names;
}
+
+ /**
+ * Makes MDC properties
+ */
+ public static void setupMDC(String service)
+ {
+ MDC.put("apex.service", service);
+
+ String value = StramClientUtils.getHostName();
+ MDC.put("apex.node", value == null ? "unknown" : value);
+
+ value = System.getenv(Environment.USER.key());
+ if (value != null) {
+ MDC.put("apex.user", value);
+ }
+
+ value = System.getenv(Environment.CONTAINER_ID.name());
+ if (value != null) {
+ ContainerId containerId = ConverterUtils.toContainerId(value);
+ ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
+ MDC.put("apex.containerId", containerId.toString());
+ MDC.put("apex.applicationId", applicationId.toString());
+ }
+
+ value = System.getProperty(APPLICATION_NAME.getLongName());
+ if (value != null) {
+ MDC.put("apex.application", value);
+ }
+ }
}
diff --git a/engine/src/test/java/com/datatorrent/stram/util/LoggerUtilTest.java b/engine/src/test/java/com/datatorrent/stram/util/LoggerUtilTest.java
index bc73ca8..88aec3b 100644
--- a/engine/src/test/java/com/datatorrent/stram/util/LoggerUtilTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/util/LoggerUtilTest.java
@@ -36,7 +36,9 @@
import com.google.common.collect.Maps;
import com.datatorrent.api.Context;
+import com.datatorrent.stram.client.StramClientUtils;
+import static com.datatorrent.api.Context.DAGContext.APPLICATION_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -220,14 +222,44 @@
assertNull(org.apache.log4j.Logger.getRootLogger().getAppender(name));
}
+ @Test
+ public void testSetupMDC()
+ {
+ // The test does not test MDC properties that are passed via environment variables
+
+ String appenderName = "mdcTestAppender";
+ String service = "test";
+ String application = "my application";
+ String args = "log4j.appender.mdcTestAppender=com.datatorrent.stram.util.LoggerUtilTest$TestAppender"
+ + ",log4j.appender.mdcTestAppender.layout=org.apache.log4j.PatternLayout"
+ + ",log4j.appender.mdcTestAppender.layout.ConversionPattern=%d %d{Z} [%t] %-5p (%F:%L) - %m%n";
+
+ System.setProperty(APPLICATION_NAME.getLongName(), application);
+ LoggerUtil.setupMDC(service);
+
+ LoggerUtil.addAppenders(new String[] {appenderName }, args, ",");
+ TestAppender appender = (TestAppender)org.apache.log4j.Logger.getRootLogger().getAppender(appenderName);
+
+ logger.info(args);
+ assertEquals(service, appender.mdcProperties.get("apex.service"));
+ String node = StramClientUtils.getHostName();
+ assertEquals(node == null ? "unknown" : node, appender.mdcProperties.get("apex.node"));
+ assertEquals(application, appender.mdcProperties.get("apex.application"));
+
+ LoggerUtil.removeAppender(appenderName);
+ }
+
public static class TestAppender extends ConsoleAppender
{
private String lastMessage = null;
private Level level;
+ private Map mdcProperties;
@Override
public void append(LoggingEvent event)
{
+ event.getMDCCopy();
+ mdcProperties = event.getProperties();
lastMessage = event.getRenderedMessage();
level = event.getLevel();
super.append(event);
