HADOOP-18373. IOStatisticsContext tuning (#4705)
The name of the option to enable/disable thread level statistics is
"fs.iostatistics.thread.level.enabled";
There is also an enabled() probe in IOStatisticsContext which can
be used to see if the thread level statistics is active.
Contributed by Viraj Jasani
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 7c54b32..75b19e2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -480,13 +480,13 @@
* Thread-level IOStats Support.
* {@value}
*/
- public static final String THREAD_LEVEL_IOSTATISTICS_ENABLED =
- "fs.thread.level.iostatistics.enabled";
+ public static final String IOSTATISTICS_THREAD_LEVEL_ENABLED =
+ "fs.iostatistics.thread.level.enabled";
/**
* Default value for Thread-level IOStats Support is true.
*/
- public static final boolean THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT =
+ public static final boolean IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT =
true;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java
index fb10b93..557c57e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java
@@ -80,4 +80,14 @@
IOStatisticsContextIntegration.setThreadIOStatisticsContext(
statisticsContext);
}
+
+ /**
+ * Static probe to check if the thread-level IO statistics enabled.
+ *
+ * @return if the thread-level IO statistics enabled.
+ */
+ static boolean enabled() {
+ return IOStatisticsContextIntegration.isIOStatisticsThreadLevelEnabled();
+ }
+
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java
index 483d1e4..2a394e6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java
@@ -29,8 +29,8 @@
import org.apache.hadoop.fs.impl.WeakReferenceThreadMap;
import org.apache.hadoop.fs.statistics.IOStatisticsContext;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.THREAD_LEVEL_IOSTATISTICS_ENABLED;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT;
/**
* A Utility class for IOStatisticsContext, which helps in creating and
@@ -76,8 +76,17 @@
// Work out if the current context has thread level IOStatistics enabled.
final Configuration configuration = new Configuration();
isThreadIOStatsEnabled =
- configuration.getBoolean(THREAD_LEVEL_IOSTATISTICS_ENABLED,
- THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT);
+ configuration.getBoolean(IOSTATISTICS_THREAD_LEVEL_ENABLED,
+ IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT);
+ }
+
+ /**
+ * Static probe to check if the thread-level IO statistics enabled.
+ *
+ * @return if the thread-level IO statistics enabled.
+ */
+ public static boolean isIOStatisticsThreadLevelEnabled() {
+ return isThreadIOStatsEnabled;
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java
index 19c40c6..d2ae9d7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java
@@ -153,6 +153,8 @@
* @return thread context
*/
private static IOStatisticsContext getAndResetThreadStatisticsContext() {
+ assertTrue("thread-level IOStatistics should be enabled by default",
+ IOStatisticsContext.enabled());
IOStatisticsContext context =
IOStatisticsContext.getCurrentIOStatisticsContext();
context.reset();
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
index 1525f51..600ea3a 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
@@ -178,6 +178,12 @@
<value>true</value>
</property>
+ <!-- Enable IOStatisticsContext support for Thread level. -->
+ <property>
+ <name>fs.iostatistics.thread.level.enabled</name>
+ <value>true</value>
+ </property>
+
<!--
To run these tests.
