YARN-11286. Make AsyncDispatcher#printEventDetailsExecutor thread pool parameter configurable. (#4824). Contributed by fanshilun.
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index cc2c10c..ad4bcf8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2988,6 +2988,51 @@
DEFAULT_YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN = 60;
/**
+ * Resource manager dispatcher has a thread pool that prints EventQueue,
+ * configure the corePoolSize of this thread pool,
+ * the meaning of corePoolSize is the number of threads to keep in the pool.
+ */
+ public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE =
+ YARN_PREFIX + "dispatcher.print-thread-pool.core-pool-size";
+
+ /**
+ * The minimum number of core threads for the
+ * Resource manager dispatcher is 1.
+ */
+ public static final int DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE = 1;
+
+ /**
+ * Resource manager dispatcher has a thread pool that prints EventQueue,
+ * configure the maximumPoolSize of this thread pool,
+ * the meaning of maximumPoolSize is the maximum number of threads to allow in the pool.
+ */
+ public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE =
+ YARN_PREFIX + "dispatcher.print-thread-pool.maximum-pool-size";
+
+ /**
+ * The maximum number of core threads for the
+ * Resource manager dispatcher is 5.
+ */
+ public static final int DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE = 5;
+
+ /**
+ * Resource manager dispatcher has a thread pool that prints EventQueue,
+ * configure the keepAliveTime of this thread pool,
+ * The meaning of keepAliveTime is as follows
+ * when the number of threads is greater than the core,
+ * this is the maximum time that excess idle threads will wait for new tasks before terminating.
+ */
+ public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME =
+ YARN_PREFIX + "dispatcher.print-thread-pool.keep-alive-time";
+
+ /**
+ * The keep alive time of core threads for the
+ * Resource manager dispatcher is 10s.
+ */
+ public static final long DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME =
+ 10*1000; // 10s
+
+ /**
* CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
* entries
*/
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 1c4ed24..eee927c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -191,8 +191,21 @@
.build();
// Thread pool for async print event details,
// to prevent wasting too much time for RM.
+ int numCorePoolSizeThreads = getConfig().getInt(
+ YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE,
+ YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE);
+
+ int numMaximumPoolSizeThreads = getConfig().getInt(
+ YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE,
+ YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE);
+
+ long keepAliveTime =
+ conf.getTimeDuration(YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME,
+ YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME,
+ TimeUnit.SECONDS);
+
printEventDetailsExecutor = new ThreadPoolExecutor(
- 1, 5, 10, TimeUnit.SECONDS,
+ numCorePoolSizeThreads, numMaximumPoolSizeThreads, keepAliveTime, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(), threadFactory);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 9d95fd4..fb3af3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -134,6 +134,41 @@
</property>
<property>
+ <description>
+ Resource manager dispatcher has a thread pool that prints EventQueue,
+ configure the corePoolSize of this thread pool,
+ the meaning of corePoolSize is the number of threads to keep in the pool.
+ the default value is 1.
+ </description>
+ <name>yarn.dispatcher.print-thread-pool.core-pool-size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <description>
+ Resource manager dispatcher has a thread pool that prints EventQueue,
+ configure the maximumPoolSize of this thread pool,
+ the meaning of maximumPoolSize is the maximum number of threads to allow in the pool.
+ the default value is 5.
+ </description>
+ <name>yarn.dispatcher.print-thread-pool.maximum-pool-size</name>
+ <value>5</value>
+ </property>
+
+ <property>
+ <description>
+ Resource manager dispatcher has a thread pool that prints EventQueue,
+ configure the keepAliveTime of this thread pool,
+ The meaning of keepAliveTime is as follows
+ when the number of threads is greater than the core,
+ this is the maximum time that excess idle threads will wait for new tasks before terminating.
+ the default value is 10s.
+ </description>
+ <name>yarn.dispatcher.print-thread-pool.keep-alive-time</name>
+ <value>10s</value>
+ </property>
+
+ <property>
<description>The expiry interval for application master reporting.</description>
<name>yarn.am.liveness-monitor.expiry-interval-ms</name>
<value>600000</value>
