AMBARI-20276 : Perf - AMS scale test for 3000 node cluster. (avijayan)
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java
index 203a88b..cd1dd1b 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java
@@ -62,6 +62,7 @@
public LoadRunner(String hostName,
int threadCount,
String metricsHostName,
+ int minHostIndex,
int collectIntervalMillis,
int sendIntervalMillis,
boolean createMaster) {
@@ -70,24 +71,24 @@
this.timer = Executors.newScheduledThreadPool(1);
this.sendIntervalMillis = sendIntervalMillis;
- workers = prepareWorkers(hostName, threadCount, metricsHostName, createMaster);
+ workers = prepareWorkers(hostName, threadCount, metricsHostName, createMaster, minHostIndex);
}
private Collection<Callable<String>> prepareWorkers(String hostName,
int threadCount,
String metricsHost,
- Boolean createMaster) {
+ Boolean createMaster, int minHostIndex) {
Collection<Callable<String>> senderWorkers =
new ArrayList<Callable<String>>(threadCount);
- int startIndex = 0;
+ int startIndex = minHostIndex;
if (createMaster) {
- String simHost = hostName + "0";
+ String simHost = hostName + startIndex;
addMetricsWorkers(senderWorkers, simHost, metricsHost, MASTER_APPS);
startIndex++;
}
- for (int i = startIndex; i < threadCount; i++) {
+ for (int i = startIndex; i < threadCount + minHostIndex; i++) {
String simHost = hostName + i;
addMetricsWorkers(senderWorkers, simHost, metricsHost, SLAVE_APPS);
}
@@ -150,7 +151,7 @@
public static void main(String[] args) {
LoadRunner runner =
- new LoadRunner("local", 2, "metrics", 10000, 20000, false);
+ new LoadRunner("local", 0, "metrics", 0, 10000, 20000, false);
runner.start();
}
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java
index 09db9b5..b64f542 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java
@@ -43,6 +43,7 @@
mapArgs.get("hostName"),
Integer.valueOf(mapArgs.get("numberOfHosts")),
mapArgs.get("metricsHostName"),
+ mapArgs.get("minHostIndex") == null ? 0 : Integer.valueOf(mapArgs.get("minHostIndex")),
Integer.valueOf(mapArgs.get("collectInterval")),
Integer.valueOf(mapArgs.get("sendInterval")),
Boolean.valueOf(mapArgs.get("master"))
@@ -60,6 +61,7 @@
mapArgs.get("hostName"),
Integer.valueOf(mapArgs.get("numberOfHosts")),
mapArgs.get("metricsHostName"),
+ mapArgs.get("minHostIndex") == null ? 0 : Integer.valueOf(mapArgs.get("minHostIndex")),
Integer.valueOf(mapArgs.get("collectInterval")),
Integer.valueOf(mapArgs.get("sendInterval")),
Boolean.valueOf(mapArgs.get("master"))
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java
index a130171..56c3571 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java
@@ -27,9 +27,11 @@
MASTER_HBASE("hbase"),
SLAVE_HBASE("hbase"),
NIMBUS("nimbus"),
+ HIVEMETASTORE("hivemetastore"),
+ HIVESERVER2("hiveserver2"),
KAFKA_BROKER("kafka_broker");
- public static final AppID[] MASTER_APPS = {HOST, NAMENODE, RESOURCEMANAGER, MASTER_HBASE, KAFKA_BROKER, NIMBUS};
+ public static final AppID[] MASTER_APPS = {HOST, NAMENODE, RESOURCEMANAGER, MASTER_HBASE, KAFKA_BROKER, NIMBUS, HIVEMETASTORE, HIVESERVER2};
public static final AppID[] SLAVE_APPS = {HOST, DATANODE, NODEMANAGER, SLAVE_HBASE};
private String id;
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index 80d451e..0d5042f 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -260,6 +260,9 @@
public static final String TIMELINE_METRICS_APPS_BLACKLIST =
"timeline.metrics.apps.blacklist";
+ public static final String TIMELINE_METRICS_APPS_WHITELIST =
+ "timeline.metrics.apps.whitelist";
+
public static final String HBASE_BLOCKING_STORE_FILES =
"hbase.hstore.blockingStoreFiles";
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java
index 0fe979e..1446ec2 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java
@@ -38,11 +38,13 @@
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_WHITELIST_FILE;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_APPS_BLACKLIST;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_APPS_WHITELIST;
public class TimelineMetricsFilter {
private static Set<String> whitelistedMetrics;
private static Set<Pattern> whitelistedMetricPatterns;
+ private static Set<String> whitelistedApps;
private static Set<String> blacklistedApps;
private static String patternPrefix = "._p_";
private static Set<String> amshbaseWhitelist;
@@ -62,6 +64,7 @@
whitelistedMetrics = new HashSet<String>();
whitelistedMetricPatterns = new HashSet<Pattern>();
blacklistedApps = new HashSet<>();
+ whitelistedApps = new HashSet<>();
amshbaseWhitelist = new HashSet<>();
String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, "");
@@ -77,6 +80,14 @@
LOG.info("Blacklisted apps : " + blacklistedApps.toString());
}
+ String appsWhitelist = metricsConf.get(TIMELINE_METRICS_APPS_WHITELIST, "");
+ if (!StringUtils.isEmpty(appsWhitelist)) {
+ for (String app : appsWhitelist.split(",")) {
+ whitelistedApps.add(app);
+ }
+ LOG.info("Whitelisted apps : " + whitelistedApps.toString());
+ }
+
amshbaseWhitelist = configuration.getAmshbaseWhitelist();
if (CollectionUtils.isNotEmpty(amshbaseWhitelist)) {
LOG.info("Whitelisting " + amshbaseWhitelist.size() + " ams-hbase metrics");
@@ -119,28 +130,36 @@
public static boolean acceptMetric(TimelineMetric metric) {
- // App takes precedence.
- if (CollectionUtils.isNotEmpty(blacklistedApps) && blacklistedApps.contains(metric.getAppId())) {
+ String appId = metric.getAppId();
+ String metricName = metric.getMetricName();
+ // App Blacklisting
+ if (CollectionUtils.isNotEmpty(blacklistedApps) && blacklistedApps.contains(appId)) {
return false;
}
//Special Case appId = ams-hbase whitelisting.
- if ("ams-hbase".equals(metric.getAppId()) && CollectionUtils.isNotEmpty(amshbaseWhitelist)) {
+ if ("ams-hbase".equals(appId) && CollectionUtils.isNotEmpty(amshbaseWhitelist)) {
return amshbaseWhitelist.contains(metric.getMetricName());
}
+ // App Whitelisting
+ if (CollectionUtils.isNotEmpty(whitelistedApps) && whitelistedApps.contains(appId)) {
+ return true;
+ }
+
+ // Metric Whitelisting
if (CollectionUtils.isEmpty(whitelistedMetrics) && CollectionUtils.isEmpty(whitelistedMetricPatterns)) {
return true;
}
- if (whitelistedMetrics.contains(metric.getMetricName())) {
+ if (whitelistedMetrics.contains(metricName)) {
return true;
}
for (Pattern p : whitelistedMetricPatterns) {
- Matcher m = p.matcher(metric.getMetricName());
+ Matcher m = p.matcher(metricName);
if (m.find()) {
- whitelistedMetrics.add(metric.getMetricName());
+ whitelistedMetrics.add(metricName);
return true;
}
}
diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMS-HBASE.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMS-HBASE.dat
index 63ac9f3..de4068b 100644
--- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMS-HBASE.dat
+++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMS-HBASE.dat
@@ -1,18 +1,245 @@
-dfs.FSNamesystem.MissingReplOneBlocks
-dfs.FSNamesystem.TransactionsSinceLastCheckpoint
-dfs.FSNamesystem.MillisSinceLastLoadedEdits
-dfs.FSNamesystem.SnapshottableDirectories
-master.Master.QueueCallTime_median
-dfs.FSNamesystem.LastCheckpointTime
-dfs.FSNamesystem.TotalFiles
-dfs.FSNamesystem.ExpiredHeartbeats
-dfs.FSNamesystem.PostponedMisreplicatedBlocks
-dfs.FSNamesystem.LastWrittenTransactionId
+regionserver.WAL.SyncTime_min
+regionserver.WAL.SyncTime_num_ops
+regionserver.WAL.appendCount
+regionserver.WAL.slowAppendCount
+jvm.JvmMetrics.GcCountParNew
+jvm.JvmMetrics.GcTimeMillis
+jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep
+jvm.JvmMetrics.GcTimeMillisParNew
+ugi.UgiMetrics.GetGroupsAvgTime
+ugi.UgiMetrics.GetGroupsNumOps
+ugi.UgiMetrics.LoginFailureNumOps
+ugi.UgiMetrics.LoginSuccessAvgTime
+ugi.UgiMetrics.LoginSuccessNumOps
+ugi.UgiMetrics.LoginFailureAvgTime
+jvm.JvmMetrics.LogError
+jvm.JvmMetrics.LogFatal
+jvm.JvmMetrics.LogInfo
+jvm.JvmMetrics.LogWarn
jvm.JvmMetrics.MemHeapCommittedM
-dfs.FSNamesystem.Snapshots
-dfs.FSNamesystem.TransactionsSinceLastLogRoll
-master.Server.averageLoad
+jvm.JvmMetrics.MemHeapMaxM
jvm.JvmMetrics.MemHeapUsedM
+jvm.JvmMetrics.MemMaxM
+jvm.JvmMetrics.MemNonHeapCommittedM
+jvm.JvmMetrics.MemNonHeapMaxM
+jvm.JvmMetrics.MemNonHeapUsedM
+jvm.JvmMetrics.ThreadsBlocked
+jvm.JvmMetrics.ThreadsNew
+jvm.JvmMetrics.ThreadsRunnable
+jvm.JvmMetrics.ThreadsTerminated
+jvm.JvmMetrics.ThreadsTimedWaiting
+master.AssignmentManger.Assign_75th_percentile
+master.AssignmentManger.Assign_95th_percentile
+master.AssignmentManger.Assign_99th_percentile
+master.AssignmentManger.Assign_max
+master.AssignmentManger.Assign_mean
+master.AssignmentManger.Assign_median
+master.AssignmentManger.Assign_min
+jvm.JvmMetrics.ThreadsWaiting
+master.AssignmentManger.Assign_num_ops
+master.AssignmentManger.BulkAssign_75th_percentile
+master.AssignmentManger.BulkAssign_95th_percentile
+master.AssignmentManger.BulkAssign_99th_percentile
+master.AssignmentManger.BulkAssign_max
+master.AssignmentManger.BulkAssign_mean
+master.AssignmentManger.BulkAssign_median
+master.AssignmentManger.BulkAssign_min
+master.AssignmentManger.BulkAssign_num_ops
master.AssignmentManger.ritCount
-dfs.FSNamesystem.PendingDataNodeMessageCount
-dfs.FSNamesystem.StaleDataNodes
\ No newline at end of file
+master.AssignmentManger.ritCountOverThreshold
+master.AssignmentManger.ritOldestAge
+master.Balancer.BalancerCluster_75th_percentile
+master.Balancer.BalancerCluster_95th_percentile
+master.Balancer.BalancerCluster_99th_percentile
+master.Balancer.BalancerCluster_max
+master.Balancer.BalancerCluster_mean
+master.Balancer.BalancerCluster_median
+master.Balancer.BalancerCluster_min
+master.Balancer.BalancerCluster_num_ops
+master.Balancer.miscInvocationCount
+master.FileSystem.HlogSplitSize_75th_percentile
+master.FileSystem.HlogSplitSize_95th_percentile
+master.FileSystem.HlogSplitSize_99th_percentile
+master.FileSystem.HlogSplitSize_max
+master.FileSystem.HlogSplitSize_mean
+master.FileSystem.HlogSplitSize_median
+master.FileSystem.HlogSplitSize_min
+master.FileSystem.HlogSplitSize_num_ops
+master.FileSystem.HlogSplitTime_75th_percentile
+master.FileSystem.HlogSplitTime_95th_percentile
+master.FileSystem.HlogSplitTime_99th_percentile
+master.FileSystem.HlogSplitTime_max
+master.FileSystem.HlogSplitTime_mean
+master.FileSystem.HlogSplitTime_median
+master.FileSystem.HlogSplitTime_min
+master.FileSystem.HlogSplitTime_num_ops
+master.FileSystem.MetaHlogSplitSize_75th_percentile
+master.FileSystem.MetaHlogSplitSize_95th_percentile
+master.FileSystem.MetaHlogSplitSize_99th_percentile
+master.FileSystem.MetaHlogSplitSize_max
+master.FileSystem.MetaHlogSplitSize_mean
+master.FileSystem.MetaHlogSplitSize_median
+master.FileSystem.MetaHlogSplitSize_min
+master.FileSystem.MetaHlogSplitSize_num_ops
+master.FileSystem.MetaHlogSplitTime_75th_percentile
+master.FileSystem.MetaHlogSplitTime_95th_percentile
+master.FileSystem.MetaHlogSplitTime_99th_percentile
+master.FileSystem.MetaHlogSplitTime_max
+master.FileSystem.MetaHlogSplitTime_mean
+master.FileSystem.MetaHlogSplitTime_median
+master.FileSystem.MetaHlogSplitTime_min
+master.FileSystem.MetaHlogSplitTime_num_ops
+master.Server.averageLoad
+master.Server.clusterRequests
+master.Server.masterActiveTime
+master.Server.masterStartTime
+master.Server.numDeadRegionServers
+master.Server.numRegionServers
+metricssystem.MetricsSystem.DroppedPubAll
+metricssystem.MetricsSystem.NumActiveSinks
+ipc.IPC.ProcessCallTime_75th_percentile
+ipc.IPC.ProcessCallTime_95th_percentile
+metricssystem.MetricsSystem.NumActiveSources
+metricssystem.MetricsSystem.NumAllSinks
+ipc.IPC.ProcessCallTime_99th_percentile
+metricssystem.MetricsSystem.NumAllSources
+metricssystem.MetricsSystem.PublishAvgTime
+metricssystem.MetricsSystem.PublishNumOps
+ipc.IPC.ProcessCallTime_max
+ipc.IPC.ProcessCallTime_mean
+metricssystem.MetricsSystem.Sink_timelineAvgTime
+ipc.IPC.ProcessCallTime_median
+metricssystem.MetricsSystem.Sink_timelineDropped
+metricssystem.MetricsSystem.Sink_timelineNumOps
+ipc.IPC.ProcessCallTime_num_ops
+metricssystem.MetricsSystem.Sink_timelineQsize
+metricssystem.MetricsSystem.SnapshotAvgTime
+ipc.IPC.QueueCallTime_95th_percentile
+metricssystem.MetricsSystem.SnapshotNumOps
+ipc.IPC.ProcessCallTime_min
+ipc.IPC.QueueCallTime_75th_percentile
+ipc.IPC.QueueCallTime_99th_percentile
+ipc.IPC.QueueCallTime_max
+ipc.IPC.QueueCallTime_mean
+ipc.IPC.QueueCallTime_median
+ipc.IPC.QueueCallTime_min
+regionserver.Server.Append_75th_percentile
+regionserver.Server.Append_95th_percentile
+ipc.IPC.QueueCallTime_num_ops
+ipc.IPC.authenticationFailures
+regionserver.Server.Append_99th_percentile
+regionserver.Server.Append_max
+ipc.IPC.authenticationSuccesses
+regionserver.Server.Append_mean
+regionserver.Server.Append_median
+regionserver.Server.Append_min
+regionserver.Server.Append_num_ops
+regionserver.Server.Delete_75th_percentile
+regionserver.Server.Delete_95th_percentile
+ipc.IPC.authorizationFailures
+regionserver.Server.Delete_99th_percentile
+regionserver.Server.Delete_max
+regionserver.Server.Delete_mean
+regionserver.Server.Delete_median
+regionserver.Server.Delete_min
+regionserver.Server.Delete_num_ops
+ipc.IPC.authorizationSuccesses
+ipc.IPC.numActiveHandler
+ipc.IPC.numCallsInGeneralQueue
+regionserver.Server.Get_75th_percentile
+regionserver.Server.Get_95th_percentile
+regionserver.Server.Get_99th_percentile
+regionserver.Server.Get_max
+regionserver.Server.Get_mean
+regionserver.Server.Get_median
+ipc.IPC.numCallsInPriorityQueue
+regionserver.Server.Get_min
+regionserver.Server.Get_num_ops
+regionserver.Server.Increment_75th_percentile
+regionserver.Server.Increment_95th_percentile
+regionserver.Server.Increment_99th_percentile
+regionserver.Server.Increment_max
+regionserver.Server.Increment_mean
+regionserver.Server.Increment_median
+ipc.IPC.numCallsInReplicationQueue
+ipc.IPC.numOpenConnections
+regionserver.Server.Increment_min
+regionserver.Server.Increment_num_ops
+ipc.IPC.queueSize
+regionserver.Server.Mutate_75th_percentile
+regionserver.Server.Mutate_95th_percentile
+regionserver.Server.Mutate_99th_percentile
+regionserver.Server.Mutate_max
+regionserver.Server.Mutate_mean
+regionserver.Server.Mutate_median
+ipc.IPC.receivedBytes
+regionserver.Server.Mutate_min
+regionserver.Server.Mutate_num_ops
+regionserver.Server.Replay_75th_percentile
+regionserver.Server.Replay_95th_percentile
+regionserver.Server.Replay_99th_percentile
+regionserver.Server.Replay_max
+regionserver.Server.Replay_mean
+regionserver.Server.Replay_median
+ipc.IPC.sentBytes
+jvm.JvmMetrics.GcCount
+regionserver.Server.Replay_min
+regionserver.Server.Replay_num_ops
+regionserver.Server.blockCacheCount
+regionserver.Server.blockCacheEvictionCount
+regionserver.Server.blockCacheExpressHitPercent
+regionserver.Server.blockCacheFreeSize
+regionserver.Server.blockCacheHitCount
+regionserver.Server.blockCacheMissCount
+regionserver.Server.blockCacheSize
+regionserver.Server.blockCountHitPercent
+regionserver.Server.checkMutateFailedCount
+regionserver.Server.checkMutatePassedCount
+regionserver.Server.compactionQueueLength
+regionserver.Server.flushQueueLength
+jvm.JvmMetrics.GcCountConcurrentMarkSweep
+regionserver.Server.hlogFileCount
+regionserver.Server.hlogFileSize
+regionserver.Server.memStoreSize
+regionserver.Server.mutationsWithoutWALCount
+regionserver.Server.mutationsWithoutWALSize
+regionserver.Server.percentFilesLocal
+regionserver.Server.readRequestCount
+regionserver.Server.regionCount
+regionserver.Server.regionServerStartTime
+regionserver.Server.slowAppendCount
+regionserver.Server.slowDeleteCount
+regionserver.Server.slowGetCount
+regionserver.Server.slowIncrementCount
+regionserver.Server.slowPutCount
+regionserver.Server.staticBloomSize
+regionserver.Server.staticIndexSize
+regionserver.Server.storeCount
+regionserver.Server.storeFileCount
+regionserver.Server.storeFileIndexSize
+regionserver.Server.storeFileSize
+regionserver.Server.totalRequestCount
+regionserver.Server.updatesBlockedTime
+regionserver.Server.writeRequestCount
+regionserver.WAL.AppendSize_75th_percentile
+regionserver.WAL.AppendSize_95th_percentile
+regionserver.WAL.AppendSize_99th_percentile
+regionserver.WAL.AppendSize_max
+regionserver.WAL.AppendSize_mean
+regionserver.WAL.AppendSize_median
+regionserver.WAL.SyncTime_median
+regionserver.WAL.AppendSize_min
+regionserver.WAL.AppendSize_num_ops
+regionserver.WAL.SyncTime_max
+regionserver.WAL.AppendTime_75th_percentile
+regionserver.WAL.AppendTime_95th_percentile
+regionserver.WAL.AppendTime_99th_percentile
+regionserver.WAL.AppendTime_max
+regionserver.WAL.SyncTime_95th_percentile
+regionserver.WAL.AppendTime_mean
+regionserver.WAL.AppendTime_median
+regionserver.WAL.AppendTime_min
+regionserver.WAL.AppendTime_num_ops
+regionserver.WAL.SyncTime_75th_percentile
+regionserver.WAL.SyncTime_99th_percentile
+regionserver.WAL.SyncTime_mean
\ No newline at end of file
diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/DATANODE.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/DATANODE.dat
index e157630..c015865 100644
--- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/DATANODE.dat
+++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/DATANODE.dat
@@ -1,101 +1,161 @@
-| dfs.datanode.BlockChecksumOpAvgTime | datanode |
-| dfs.datanode.BlockChecksumOpNumOps | datanode |
-| dfs.datanode.BlockReportsAvgTime | datanode |
-| dfs.datanode.BlockReportsNumOps | datanode |
-| dfs.datanode.BlockVerificationFailures | datanode |
-| dfs.datanode.BlocksCached | datanode |
-| dfs.datanode.BlocksGetLocalPathInfo | datanode |
-| dfs.datanode.BlocksRead | datanode |
-| dfs.datanode.BlocksRemoved | datanode |
-| dfs.datanode.BlocksReplicated | datanode |
-| dfs.datanode.BlocksUncached | datanode |
-| dfs.datanode.BlocksVerified | datanode |
-| dfs.datanode.BlocksWritten | datanode |
-| dfs.datanode.BytesRead | datanode |
-| dfs.datanode.BytesWritten | datanode |
-| dfs.datanode.CacheReportsAvgTime | datanode |
-| dfs.datanode.CacheReportsNumOps | datanode |
-| dfs.datanode.CopyBlockOpAvgTime | datanode |
-| dfs.datanode.CopyBlockOpNumOps | datanode |
-| dfs.datanode.FlushNanosAvgTime | datanode |
-| dfs.datanode.FlushNanosNumOps | datanode |
-| dfs.datanode.FsyncCount | datanode |
-| dfs.datanode.FsyncNanosAvgTime | datanode |
-| dfs.datanode.FsyncNanosNumOps | datanode |
-| dfs.datanode.HeartbeatsAvgTime | datanode |
-| dfs.datanode.HeartbeatsNumOps | datanode |
-| dfs.datanode.PacketAckRoundTripTimeNanosAvgTime | datanode |
-| dfs.datanode.PacketAckRoundTripTimeNanosNumOps | datanode |
-| dfs.datanode.ReadBlockOpAvgTime | datanode |
-| dfs.datanode.ReadBlockOpNumOps | datanode |
-| dfs.datanode.ReadsFromLocalClient | datanode |
-| dfs.datanode.ReadsFromRemoteClient | datanode |
-| dfs.datanode.ReplaceBlockOpAvgTime | datanode |
-| dfs.datanode.ReplaceBlockOpNumOps | datanode |
-| dfs.datanode.SendDataPacketBlockedOnNetworkNanosAvgTime | datanode |
-| dfs.datanode.SendDataPacketBlockedOnNetworkNanosNumOps | datanode |
-| dfs.datanode.SendDataPacketTransferNanosAvgTime | datanode |
-| dfs.datanode.SendDataPacketTransferNanosNumOps | datanode |
-| dfs.datanode.VolumeFailures | datanode |
-| dfs.datanode.WriteBlockOpAvgTime | datanode |
-| dfs.datanode.WriteBlockOpNumOps | datanode |
-| dfs.datanode.WritesFromLocalClient | datanode |
-| dfs.datanode.WritesFromRemoteClient | datanode |
-| jvm.JvmMetrics.GcCount | datanode |
-| jvm.JvmMetrics.GcCountCopy | datanode |
-| jvm.JvmMetrics.GcCountMarkSweepCompact | datanode |
-| jvm.JvmMetrics.GcTimeMillis | datanode |
-| jvm.JvmMetrics.GcTimeMillisCopy | datanode |
-| jvm.JvmMetrics.GcTimeMillisMarkSweepCompact | datanode |
-| jvm.JvmMetrics.LogError | datanode |
-| jvm.JvmMetrics.LogFatal | datanode |
-| jvm.JvmMetrics.LogInfo | datanode |
-| jvm.JvmMetrics.LogWarn | datanode |
-| jvm.JvmMetrics.MemHeapCommittedM | datanode |
-| jvm.JvmMetrics.MemHeapMaxM | datanode |
-| jvm.JvmMetrics.MemHeapUsedM | datanode |
-| jvm.JvmMetrics.MemMaxM | datanode |
-| jvm.JvmMetrics.MemNonHeapCommittedM | datanode |
-| jvm.JvmMetrics.MemNonHeapMaxM | datanode |
-| jvm.JvmMetrics.MemNonHeapUsedM | datanode |
-| jvm.JvmMetrics.ThreadsBlocked | datanode |
-| jvm.JvmMetrics.ThreadsNew | datanode |
-| jvm.JvmMetrics.ThreadsRunnable | datanode |
-| jvm.JvmMetrics.ThreadsTerminated | datanode |
-| jvm.JvmMetrics.ThreadsTimedWaiting | datanode |
-| jvm.JvmMetrics.ThreadsWaiting | datanode |
-| metricssystem.MetricsSystem.DroppedPubAll | datanode |
-| metricssystem.MetricsSystem.NumActiveSinks | datanode |
-| metricssystem.MetricsSystem.NumActiveSources | datanode |
-| metricssystem.MetricsSystem.NumAllSinks | datanode |
-| metricssystem.MetricsSystem.NumAllSources | datanode |
-| metricssystem.MetricsSystem.PublishAvgTime | datanode |
-| metricssystem.MetricsSystem.PublishNumOps | datanode |
-| metricssystem.MetricsSystem.Sink_timelineAvgTime | datanode |
-| metricssystem.MetricsSystem.Sink_timelineDropped | datanode |
-| metricssystem.MetricsSystem.Sink_timelineNumOps | datanode |
-| metricssystem.MetricsSystem.Sink_timelineQsize | datanode |
-| metricssystem.MetricsSystem.SnapshotAvgTime | datanode |
-| metricssystem.MetricsSystem.SnapshotNumOps | datanode |
-| rpc.rpc.CallQueueLength | datanode |
-| rpc.rpc.NumOpenConnections | datanode |
-| rpc.rpc.ReceivedBytes | datanode |
-| rpc.rpc.RpcAuthenticationFailures | datanode |
-| rpc.rpc.RpcAuthenticationSuccesses | datanode |
-| rpc.rpc.RpcAuthorizationFailures | datanode |
-| rpc.rpc.RpcAuthorizationSuccesses | datanode |
-| rpc.rpc.RpcProcessingTimeAvgTime | datanode |
-| rpc.rpc.RpcProcessingTimeNumOps | datanode |
-| rpc.rpc.RpcQueueTimeAvgTime | datanode |
-| rpc.rpc.RpcQueueTimeNumOps | datanode |
-| rpc.rpc.SentBytes | datanode |
-| rpcdetailed.rpcdetailed.InitReplicaRecoveryAvgTime | datanode |
-| rpcdetailed.rpcdetailed.InitReplicaRecoveryNumOps | datanode |
-| rpcdetailed.rpcdetailed.UpdateReplicaUnderRecoveryAvgTime | datanode |
-| rpcdetailed.rpcdetailed.UpdateReplicaUnderRecoveryNumOps | datanode |
-| ugi.UgiMetrics.GetGroupsAvgTime | datanode |
-| ugi.UgiMetrics.GetGroupsNumOps | datanode |
-| ugi.UgiMetrics.LoginFailureAvgTime | datanode |
-| ugi.UgiMetrics.LoginFailureNumOps | datanode |
-| ugi.UgiMetrics.LoginSuccessAvgTime | datanode |
-| ugi.UgiMetrics.LoginSuccessNumOps | datanode |
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.CacheCapacity
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.CacheUsed
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.DfsUsed
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.EstimatedCapacityLostTotal
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.LastVolumeFailureDate
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumBlocksCached
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumBlocksFailedToCache
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumBlocksFailedToUnCache
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumFailedVolumes
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Remaining
+dfs.FsVolume.DataFileIoRateAvgTime
+dfs.FsVolume.DataFileIoRateNumOps
+dfs.FsVolume.FileIoErrorRateAvgTime
+dfs.FsVolume.FileIoErrorRateNumOps
+dfs.FsVolume.FlushIoRateAvgTime
+dfs.FsVolume.FlushIoRateNumOps
+dfs.FsVolume.MetadataOperationRateAvgTime
+dfs.FsVolume.MetadataOperationRateNumOps
+dfs.FsVolume.ReadIoRateAvgTime
+dfs.FsVolume.ReadIoRateNumOps
+dfs.FsVolume.SyncIoRateAvgTime
+dfs.FsVolume.SyncIoRateNumOps
+dfs.FsVolume.TotalDataFileIos
+dfs.FsVolume.TotalFileIoErrors
+dfs.FsVolume.TotalMetadataOperations
+dfs.FsVolume.WriteIoRateAvgTime
+dfs.FsVolume.WriteIoRateNumOps
+dfs.datanode.BlockChecksumOpAvgTime
+dfs.datanode.BlockChecksumOpNumOps
+dfs.datanode.BlockReportsAvgTime
+dfs.datanode.BlockReportsNumOps
+dfs.datanode.BlockVerificationFailures
+dfs.datanode.BlocksCached
+dfs.datanode.BlocksGetLocalPathInfo
+dfs.datanode.BlocksRead
+dfs.datanode.BlocksRemoved
+dfs.datanode.BlocksReplicated
+dfs.datanode.BlocksUncached
+dfs.datanode.BlocksVerified
+dfs.datanode.BlocksWritten
+dfs.datanode.BytesRead
+dfs.datanode.BytesWritten
+dfs.datanode.CacheReportsAvgTime
+dfs.datanode.CacheReportsNumOps
+dfs.datanode.CopyBlockOpAvgTime
+dfs.datanode.CopyBlockOpNumOps
+dfs.datanode.DataNodeActiveXceiversCount
+dfs.datanode.DatanodeNetworkErrors
+dfs.datanode.FlushNanosAvgTime
+dfs.datanode.FlushNanosNumOps
+dfs.datanode.FsyncCount
+dfs.datanode.FsyncNanosAvgTime
+dfs.datanode.FsyncNanosNumOps
+dfs.datanode.HeartbeatsAvgTime
+dfs.datanode.HeartbeatsNumOps
+dfs.datanode.HeartbeatsTotalAvgTime
+dfs.datanode.HeartbeatsTotalNumOps
+dfs.datanode.IncrementalBlockReportsAvgTime
+dfs.datanode.IncrementalBlockReportsNumOps
+dfs.datanode.LifelinesAvgTime
+dfs.datanode.LifelinesNumOps
+dfs.datanode.PacketAckRoundTripTimeNanosAvgTime
+dfs.datanode.PacketAckRoundTripTimeNanosNumOps
+dfs.datanode.RamDiskBlocksDeletedBeforeLazyPersisted
+dfs.datanode.RamDiskBlocksEvicted
+dfs.datanode.RamDiskBlocksEvictedWithoutRead
+dfs.datanode.RamDiskBlocksEvictionWindowMsAvgTime
+dfs.datanode.RamDiskBlocksEvictionWindowMsNumOps
+dfs.datanode.RamDiskBlocksLazyPersistWindowMsAvgTime
+dfs.datanode.RamDiskBlocksLazyPersistWindowMsNumOps
+dfs.datanode.RamDiskBlocksLazyPersisted
+dfs.datanode.RamDiskBlocksReadHits
+dfs.datanode.RamDiskBlocksWrite
+dfs.datanode.RamDiskBlocksWriteFallback
+dfs.datanode.RamDiskBytesLazyPersisted
+dfs.datanode.RamDiskBytesWrite
+dfs.datanode.ReadBlockOpAvgTime
+dfs.datanode.ReadBlockOpNumOps
+dfs.datanode.ReadsFromLocalClient
+dfs.datanode.ReadsFromRemoteClient
+dfs.datanode.RemoteBytesRead
+dfs.datanode.RemoteBytesWritten
+dfs.datanode.ReplaceBlockOpAvgTime
+dfs.datanode.ReplaceBlockOpNumOps
+dfs.datanode.SendDataPacketBlockedOnNetworkNanosAvgTime
+dfs.datanode.SendDataPacketBlockedOnNetworkNanosNumOps
+dfs.datanode.SendDataPacketTransferNanosAvgTime
+dfs.datanode.SendDataPacketTransferNanosNumOps
+dfs.datanode.TotalReadTime
+dfs.datanode.TotalWriteTime
+dfs.datanode.VolumeFailures
+dfs.datanode.WriteBlockOpAvgTime
+dfs.datanode.WriteBlockOpNumOps
+dfs.datanode.WritesFromLocalClient
+dfs.datanode.WritesFromRemoteClient
+jvm.JvmMetrics.GcCount
+jvm.JvmMetrics.GcCountConcurrentMarkSweep
+jvm.JvmMetrics.GcCountParNew
+jvm.JvmMetrics.GcNumInfoThresholdExceeded
+jvm.JvmMetrics.GcNumWarnThresholdExceeded
+jvm.JvmMetrics.GcTimeMillis
+jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep
+jvm.JvmMetrics.GcTimeMillisParNew
+jvm.JvmMetrics.GcTotalExtraSleepTime
+jvm.JvmMetrics.LogError
+jvm.JvmMetrics.LogFatal
+jvm.JvmMetrics.LogInfo
+jvm.JvmMetrics.LogWarn
+jvm.JvmMetrics.MemHeapCommittedM
+jvm.JvmMetrics.MemHeapMaxM
+jvm.JvmMetrics.MemHeapUsedM
+jvm.JvmMetrics.MemMaxM
+jvm.JvmMetrics.MemNonHeapCommittedM
+jvm.JvmMetrics.MemNonHeapMaxM
+jvm.JvmMetrics.MemNonHeapUsedM
+jvm.JvmMetrics.ThreadsBlocked
+jvm.JvmMetrics.ThreadsNew
+jvm.JvmMetrics.ThreadsRunnable
+jvm.JvmMetrics.ThreadsTerminated
+jvm.JvmMetrics.ThreadsTimedWaiting
+jvm.JvmMetrics.ThreadsWaiting
+metricssystem.MetricsSystem.DroppedPubAll
+metricssystem.MetricsSystem.NumActiveSinks
+metricssystem.MetricsSystem.NumActiveSources
+metricssystem.MetricsSystem.NumAllSinks
+metricssystem.MetricsSystem.NumAllSources
+metricssystem.MetricsSystem.PublishAvgTime
+metricssystem.MetricsSystem.PublishNumOps
+metricssystem.MetricsSystem.Sink_timelineAvgTime
+metricssystem.MetricsSystem.Sink_timelineDropped
+metricssystem.MetricsSystem.Sink_timelineNumOps
+metricssystem.MetricsSystem.Sink_timelineQsize
+metricssystem.MetricsSystem.SnapshotAvgTime
+metricssystem.MetricsSystem.SnapshotNumOps
+rpc.rpc.CallQueueLength
+rpc.rpc.NumOpenConnections
+rpc.rpc.ReceivedBytes
+rpc.rpc.RpcAuthenticationFailures
+rpc.rpc.RpcAuthenticationSuccesses
+rpc.rpc.RpcAuthorizationFailures
+rpc.rpc.RpcAuthorizationSuccesses
+rpc.rpc.RpcClientBackoff
+rpc.rpc.RpcProcessingTimeAvgTime
+rpc.rpc.RpcProcessingTimeNumOps
+rpc.rpc.RpcQueueTimeAvgTime
+rpc.rpc.RpcQueueTimeNumOps
+rpc.rpc.RpcSlowCalls
+rpc.rpc.SentBytes
+rpcdetailed.rpcdetailed.InitReplicaRecoveryAvgTime
+rpcdetailed.rpcdetailed.InitReplicaRecoveryNumOps
+rpcdetailed.rpcdetailed.UpdateReplicaUnderRecoveryAvgTime
+rpcdetailed.rpcdetailed.UpdateReplicaUnderRecoveryNumOps
+ugi.UgiMetrics.GetGroupsAvgTime
+ugi.UgiMetrics.GetGroupsNumOps
+ugi.UgiMetrics.LoginFailureAvgTime
+ugi.UgiMetrics.LoginFailureNumOps
+ugi.UgiMetrics.LoginSuccessAvgTime
+ugi.UgiMetrics.LoginSuccessNumOps
+ugi.UgiMetrics.RenewalFailures
+ugi.UgiMetrics.RenewalFailuresTotal
diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/FLUME_HANDLER.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/FLUME_HANDLER.dat
index bd5852f..376490a 100644
--- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/FLUME_HANDLER.dat
+++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/FLUME_HANDLER.dat
@@ -1,40 +1,17 @@
-ChannelSize._rate._min._sum
-ChannelSize._rate._sum._max
-ChannelSize._rate._min._max
-ChannelSize._rate._avg._min
-ChannelSize._rate._avg._avg
-ChannelSize._rate._max
-ChannelSize._rate._max._min
-ChannelSize._rate._max._avg
-ChannelSize._rate._avg._sum
-ChannelSize._rate._max._sum
-ChannelSize._rate._sum
-ChannelSize._rate._sum._min
-ChannelSize._rate._sum._avg
-ChannelSize._rate._min._avg
-ChannelSize._rate._min._min
-ChannelSize._rate._avg._max
-ChannelSize._rate._max._max
-ChannelSize._rate._avg
-ChannelSize._rate._min
-ChannelSize._rate._sum._sum
-EventPutSuccessCount._rate._avg._sum
-EventPutSuccessCount._rate._max._sum
-EventPutSuccessCount._rate._sum._sum
-EventPutSuccessCount._rate._max._max
-EventPutSuccessCount._rate._min._avg
-EventPutSuccessCount._rate._min._min
-EventPutSuccessCount._rate._avg._max
-EventPutSuccessCount._rate._sum._min
-EventPutSuccessCount._rate._sum._avg
-EventPutSuccessCount._rate._min._sum
-EventPutSuccessCount._rate._max
-EventPutSuccessCount._rate._max._avg
-EventPutSuccessCount._rate._avg._avg
-EventPutSuccessCount._rate._max._min
-EventPutSuccessCount._rate._avg._min
-EventPutSuccessCount._rate._avg
-EventPutSuccessCount._rate._min
-EventPutSuccessCount._rate._sum._max
-EventPutSuccessCount._rate._min._max
-EventPutSuccessCount._rate._sum
+BatchCompleteCount
+BatchEmptyCount
+BatchUnderflowCount
+ChannelCapacity
+ChannelFillPercentage
+ChannelSize
+ConnectionClosedCount
+ConnectionCreatedCount
+ConnectionFailedCount
+EventDrainAttemptCount
+EventDrainSuccessCount
+EventPutAttemptCount
+EventPutSuccessCount
+EventTakeSuccessCount
+EventTakeAttemptCount
+StartTime
+StopTime
\ No newline at end of file
diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/HIVEMETASTORE.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/HIVEMETASTORE.dat
new file mode 100644
index 0000000..8b7a893
--- /dev/null
+++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/HIVEMETASTORE.dat
@@ -0,0 +1,181 @@
+default.General.active_calls_api_create_table
+default.General.active_calls_api_drop_table
+default.General.active_calls_api_get_all_databases
+default.General.active_calls_api_get_database
+default.General.active_calls_api_get_functions
+default.General.active_calls_api_get_table
+default.General.active_calls_api_get_tables
+default.General.api_create_table_15min_rate
+default.General.api_create_table_1min_rate
+default.General.api_create_table_5min_rate
+default.General.api_create_table_75thpercentile
+default.General.api_create_table_95thpercentile
+default.General.api_create_table_98thpercentile
+default.General.api_create_table_999thpercentile
+default.General.api_create_table_99thpercentile
+default.General.api_create_table_count
+default.General.api_create_table_max
+default.General.api_create_table_mean
+default.General.api_create_table_mean_rate
+default.General.api_create_table_median
+default.General.api_create_table_min
+default.General.api_create_table_stddev
+default.General.api_drop_table_15min_rate
+default.General.api_drop_table_1min_rate
+default.General.api_drop_table_5min_rate
+default.General.api_drop_table_75thpercentile
+default.General.api_drop_table_95thpercentile
+default.General.api_drop_table_98thpercentile
+default.General.api_drop_table_999thpercentile
+default.General.api_drop_table_99thpercentile
+default.General.api_drop_table_count
+default.General.api_drop_table_max
+default.General.api_drop_table_mean
+default.General.api_drop_table_mean_rate
+default.General.api_drop_table_median
+default.General.api_drop_table_min
+default.General.api_drop_table_stddev
+default.General.api_get_all_databases_15min_rate
+default.General.api_get_all_databases_1min_rate
+default.General.api_get_all_databases_5min_rate
+default.General.api_get_all_databases_75thpercentile
+default.General.api_get_all_databases_95thpercentile
+default.General.api_get_all_databases_98thpercentile
+default.General.api_get_all_databases_999thpercentile
+default.General.api_get_all_databases_99thpercentile
+default.General.api_get_all_databases_count
+default.General.api_get_all_databases_max
+default.General.api_get_all_databases_mean
+default.General.api_get_all_databases_mean_rate
+default.General.api_get_all_databases_median
+default.General.api_get_all_databases_min
+default.General.api_get_all_databases_stddev
+default.General.api_get_database_15min_rate
+default.General.api_get_database_1min_rate
+default.General.api_get_database_5min_rate
+default.General.api_get_database_75thpercentile
+default.General.api_get_database_95thpercentile
+default.General.api_get_database_98thpercentile
+default.General.api_get_database_999thpercentile
+default.General.api_get_database_99thpercentile
+default.General.api_get_database_count
+default.General.api_get_database_max
+default.General.api_get_database_mean
+default.General.api_get_database_mean_rate
+default.General.api_get_database_median
+default.General.api_get_database_min
+default.General.api_get_database_stddev
+default.General.api_get_functions_15min_rate
+default.General.api_get_functions_1min_rate
+default.General.api_get_functions_5min_rate
+default.General.api_get_functions_75thpercentile
+default.General.api_get_functions_95thpercentile
+default.General.api_get_functions_98thpercentile
+default.General.api_get_functions_999thpercentile
+default.General.api_get_functions_99thpercentile
+default.General.api_get_functions_count
+default.General.api_get_functions_max
+default.General.api_get_functions_mean
+default.General.api_get_functions_mean_rate
+default.General.api_get_functions_median
+default.General.api_get_functions_min
+default.General.api_get_functions_stddev
+default.General.api_get_table_15min_rate
+default.General.api_get_table_1min_rate
+default.General.api_get_table_5min_rate
+default.General.api_get_table_75thpercentile
+default.General.api_get_table_95thpercentile
+default.General.api_get_table_98thpercentile
+default.General.api_get_table_999thpercentile
+default.General.api_get_table_99thpercentile
+default.General.api_get_table_count
+default.General.api_get_table_max
+default.General.api_get_table_mean
+default.General.api_get_table_mean_rate
+default.General.api_get_table_median
+default.General.api_get_table_min
+default.General.api_get_table_stddev
+default.General.api_get_tables_15min_rate
+default.General.api_get_tables_1min_rate
+default.General.api_get_tables_5min_rate
+default.General.api_get_tables_75thpercentile
+default.General.api_get_tables_95thpercentile
+default.General.api_get_tables_98thpercentile
+default.General.api_get_tables_999thpercentile
+default.General.api_get_tables_99thpercentile
+default.General.api_get_tables_count
+default.General.api_get_tables_max
+default.General.api_get_tables_mean
+default.General.api_get_tables_mean_rate
+default.General.api_get_tables_median
+default.General.api_get_tables_min
+default.General.api_get_tables_stddev
+default.General.buffers.direct.capacity
+default.General.buffers.direct.count
+default.General.buffers.direct.used
+default.General.buffers.mapped.capacity
+default.General.buffers.mapped.count
+default.General.buffers.mapped.used
+default.General.classLoading.loaded
+default.General.classLoading.unloaded
+default.General.create_total_count_tables
+default.General.delete_total_count_tables
+default.General.gc.PS-MarkSweep.count
+default.General.gc.PS-MarkSweep.time
+default.General.gc.PS-Scavenge.count
+default.General.gc.PS-Scavenge.time
+default.General.init_total_count_dbs
+default.General.init_total_count_partitions
+default.General.init_total_count_tables
+default.General.jvm.pause.extraSleepTime
+default.General.memory.heap.committed
+default.General.memory.heap.init
+default.General.memory.heap.max
+default.General.memory.heap.usage
+default.General.memory.heap.used
+default.General.memory.non-heap.committed
+default.General.memory.non-heap.init
+default.General.memory.non-heap.max
+default.General.memory.non-heap.usage
+default.General.memory.non-heap.used
+default.General.memory.pools.Code-Cache.usage
+default.General.memory.pools.Compressed-Class-Space.usage
+default.General.memory.pools.Metaspace.usage
+default.General.memory.pools.PS-Eden-Space.usage
+default.General.memory.pools.PS-Old-Gen.usage
+default.General.memory.pools.PS-Survivor-Space.usage
+default.General.memory.total.committed
+default.General.memory.total.init
+default.General.memory.total.max
+default.General.memory.total.used
+default.General.open_connections
+default.General.threads.blocked.count
+default.General.threads.count
+default.General.threads.daemon.count
+default.General.threads.deadlock.count
+default.General.threads.new.count
+default.General.threads.runnable.count
+default.General.threads.terminated.count
+default.General.threads.timed_waiting.count
+default.General.threads.waiting.count
+metricssystem.MetricsSystem.DroppedPubAll
+metricssystem.MetricsSystem.NumActiveSinks
+metricssystem.MetricsSystem.NumActiveSources
+metricssystem.MetricsSystem.NumAllSinks
+metricssystem.MetricsSystem.NumAllSources
+metricssystem.MetricsSystem.PublishAvgTime
+metricssystem.MetricsSystem.PublishNumOps
+metricssystem.MetricsSystem.Sink_timelineAvgTime
+metricssystem.MetricsSystem.Sink_timelineDropped
+metricssystem.MetricsSystem.Sink_timelineNumOps
+metricssystem.MetricsSystem.Sink_timelineQsize
+metricssystem.MetricsSystem.SnapshotAvgTime
+metricssystem.MetricsSystem.SnapshotNumOps
+ugi.UgiMetrics.GetGroupsAvgTime
+ugi.UgiMetrics.GetGroupsNumOps
+ugi.UgiMetrics.LoginFailureAvgTime
+ugi.UgiMetrics.LoginFailureNumOps
+ugi.UgiMetrics.LoginSuccessAvgTime
+ugi.UgiMetrics.LoginSuccessNumOps
+ugi.UgiMetrics.RenewalFailures
+ugi.UgiMetrics.RenewalFailuresTotal
\ No newline at end of file
diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/HIVESERVER2.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/HIVESERVER2.dat
new file mode 100644
index 0000000..c48b512
--- /dev/null
+++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/HIVESERVER2.dat
@@ -0,0 +1,117 @@
+default.General.active_calls_api_get_all_databases
+default.General.active_calls_api_get_databases
+default.General.active_calls_api_get_functions
+default.General.api_get_all_databases_15min_rate
+default.General.api_get_all_databases_1min_rate
+default.General.api_get_all_databases_5min_rate
+default.General.api_get_all_databases_75thpercentile
+default.General.api_get_all_databases_95thpercentile
+default.General.api_get_all_databases_98thpercentile
+default.General.api_get_all_databases_999thpercentile
+default.General.api_get_all_databases_99thpercentile
+default.General.api_get_all_databases_count
+default.General.api_get_all_databases_max
+default.General.api_get_all_databases_mean
+default.General.api_get_all_databases_mean_rate
+default.General.api_get_all_databases_median
+default.General.api_get_all_databases_min
+default.General.api_get_all_databases_stddev
+default.General.api_get_databases_15min_rate
+default.General.api_get_databases_1min_rate
+default.General.api_get_databases_5min_rate
+default.General.api_get_databases_75thpercentile
+default.General.api_get_databases_95thpercentile
+default.General.api_get_databases_98thpercentile
+default.General.api_get_databases_999thpercentile
+default.General.api_get_databases_99thpercentile
+default.General.api_get_databases_count
+default.General.api_get_databases_max
+default.General.api_get_databases_mean
+default.General.api_get_databases_mean_rate
+default.General.api_get_databases_median
+default.General.api_get_databases_min
+default.General.api_get_databases_stddev
+default.General.api_get_functions_15min_rate
+default.General.api_get_functions_1min_rate
+default.General.api_get_functions_5min_rate
+default.General.api_get_functions_75thpercentile
+default.General.api_get_functions_95thpercentile
+default.General.api_get_functions_98thpercentile
+default.General.api_get_functions_999thpercentile
+default.General.api_get_functions_99thpercentile
+default.General.api_get_functions_count
+default.General.api_get_functions_max
+default.General.api_get_functions_mean
+default.General.api_get_functions_mean_rate
+default.General.api_get_functions_median
+default.General.api_get_functions_min
+default.General.api_get_functions_stddev
+default.General.buffers.direct.capacity
+default.General.buffers.direct.count
+default.General.buffers.direct.used
+default.General.buffers.mapped.capacity
+default.General.buffers.mapped.count
+default.General.buffers.mapped.used
+default.General.classLoading.loaded
+default.General.classLoading.unloaded
+default.General.exec_async_pool_size
+default.General.exec_async_queue_size
+default.General.gc.PS-MarkSweep.count
+default.General.gc.PS-MarkSweep.time
+default.General.gc.PS-Scavenge.count
+default.General.gc.PS-Scavenge.time
+default.General.init_total_count_dbs
+default.General.init_total_count_partitions
+default.General.init_total_count_tables
+default.General.jvm.pause.extraSleepTime
+default.General.memory.heap.committed
+default.General.memory.heap.init
+default.General.memory.heap.max
+default.General.memory.heap.usage
+default.General.memory.heap.used
+default.General.memory.non-heap.committed
+default.General.memory.non-heap.init
+default.General.memory.non-heap.max
+default.General.memory.non-heap.usage
+default.General.memory.non-heap.used
+default.General.memory.pools.Code-Cache.usage
+default.General.memory.pools.Compressed-Class-Space.usage
+default.General.memory.pools.Metaspace.usage
+default.General.memory.pools.PS-Eden-Space.usage
+default.General.memory.pools.PS-Old-Gen.usage
+default.General.memory.pools.PS-Survivor-Space.usage
+default.General.memory.total.committed
+default.General.memory.total.init
+default.General.memory.total.max
+default.General.memory.total.used
+default.General.open_connections
+default.General.threads.blocked.count
+default.General.threads.count
+default.General.threads.daemon.count
+default.General.threads.deadlock.count
+default.General.threads.new.count
+default.General.threads.runnable.count
+default.General.threads.terminated.count
+default.General.threads.timed_waiting.count
+default.General.threads.waiting.count
+metricssystem.MetricsSystem.DroppedPubAll
+metricssystem.MetricsSystem.NumActiveSinks
+metricssystem.MetricsSystem.NumActiveSources
+metricssystem.MetricsSystem.NumAllSinks
+metricssystem.MetricsSystem.NumAllSources
+metricssystem.MetricsSystem.PublishAvgTime
+metricssystem.MetricsSystem.PublishNumOps
+metricssystem.MetricsSystem.Sink_timelineAvgTime
+metricssystem.MetricsSystem.Sink_timelineDropped
+metricssystem.MetricsSystem.Sink_timelineNumOps
+metricssystem.MetricsSystem.Sink_timelineQsize
+metricssystem.MetricsSystem.SnapshotAvgTime
+metricssystem.MetricsSystem.SnapshotNumOps
+ugi.UgiMetrics.GetGroupsAvgTime
+ugi.UgiMetrics.GetGroupsNumOps
+ugi.UgiMetrics.LoginFailureAvgTime
+ugi.UgiMetrics.LoginFailureNumOps
+ugi.UgiMetrics.LoginSuccessAvgTime
+ugi.UgiMetrics.LoginSuccessNumOps
+ugi.UgiMetrics.RenewalFailures
+ugi.UgiMetrics.RenewalFailuresTotal
\ No newline at end of file
diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/HOST.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/HOST.dat
index 9295692..3758140 100644
--- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/HOST.dat
+++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/HOST.dat
@@ -1,13 +1,17 @@
+boottime
bytes_in
bytes_out
cpu_idle
cpu_intr
cpu_nice
+cpu_num
cpu_sintr
+cpu_steal
cpu_system
cpu_user
cpu_wio
disk_free
+disk_num
disk_percent
disk_total
disk_used
@@ -18,7 +22,34 @@
mem_cached
mem_free
mem_shared
+mem_total
+mem_used
pkts_in
pkts_out
+proc_run
+proc_total
+read_bps
+read_bytes
+read_count
+read_time
+sdisk_vda1_read_bytes
+sdisk_vda1_read_count
+sdisk_vda1_read_time
+sdisk_vda1_write_bytes
+sdisk_vda1_write_count
+sdisk_vda1_write_time
+sdisk_vdb_read_bytes
+sdisk_vdb_read_count
+sdisk_vdb_read_time
+sdisk_vdb_write_bytes
+sdisk_vdb_write_count
+sdisk_vdb_write_time
swap_free
+swap_in
+swap_out
swap_total
+swap_used
+write_bps
+write_bytes
+write_count
+write_time
diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/NAMENODE.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/NAMENODE.dat
index 6e98a9c..ad35c45 100644
--- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/NAMENODE.dat
+++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/NAMENODE.dat
@@ -31,8 +31,13 @@
dfs.FSNamesystem.FilesTotal
dfs.FSNamesystem.LastCheckpointTime
dfs.FSNamesystem.LastWrittenTransactionId
+dfs.FSNamesystem.LockQueueLength
dfs.FSNamesystem.MillisSinceLastLoadedEdits
dfs.FSNamesystem.MissingBlocks
+dfs.FSNamesystem.MissingReplOneBlocks
+dfs.FSNamesystem.NumActiveClients
+dfs.FSNamesystem.NumFilesUnderConstruction
+dfs.FSNamesystem.NumTimedOutPendingReplications
dfs.FSNamesystem.PendingDataNodeMessageCount
dfs.FSNamesystem.PendingDeletionBlocks
dfs.FSNamesystem.PendingReplicationBlocks
@@ -43,11 +48,182 @@
dfs.FSNamesystem.StaleDataNodes
dfs.FSNamesystem.TotalFiles
dfs.FSNamesystem.TotalLoad
+dfs.FSNamesystem.TotalSyncCount
dfs.FSNamesystem.TransactionsSinceLastCheckpoint
dfs.FSNamesystem.TransactionsSinceLastLogRoll
dfs.FSNamesystem.UnderReplicatedBlocks
+dfs.NNTopUserOpCounts.windowMs=1500000.op=*.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=*.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=*.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=*.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=*.user=hive.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=*.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=*.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=contentSummary.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=contentSummary.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=create.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=create.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=create.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=create.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=delete.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=delete.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=delete.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=delete.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=delete.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=getfileinfo.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=getfileinfo.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=getfileinfo.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=getfileinfo.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=getfileinfo.user=hive.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=getfileinfo.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=getfileinfo.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=listEncryptionZones.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=listEncryptionZones.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=listStatus.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=listStatus.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=listStatus.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=listStatus.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=listStatus.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=listStatus.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=mkdirs.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=mkdirs.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=mkdirs.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=mkdirs.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=mkdirs.user=hive.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=mkdirs.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=mkdirs.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=open.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=open.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=open.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=open.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=open.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=rename(options=[NONE]).TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=rename(options=[NONE]).user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=rename.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=rename.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=rename.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=rename.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=setOwner.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=setOwner.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=setPermission.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=setPermission.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=setPermission.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=setPermission.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=setReplication.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=setReplication.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=1500000.op=setTimes.TotalCount
+dfs.NNTopUserOpCounts.windowMs=1500000.op=setTimes.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=*.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=hive.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=contentSummary.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=contentSummary.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=create.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=create.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=create.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=create.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=delete.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=delete.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=delete.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=delete.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=delete.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=getfileinfo.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=getfileinfo.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=getfileinfo.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=getfileinfo.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=getfileinfo.user=hive.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=getfileinfo.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=getfileinfo.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=listEncryptionZones.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=listEncryptionZones.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=listStatus.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=listStatus.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=listStatus.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=listStatus.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=listStatus.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=listStatus.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=mkdirs.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=mkdirs.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=mkdirs.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=mkdirs.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=mkdirs.user=hive.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=mkdirs.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=mkdirs.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=open.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=open.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=open.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=open.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=open.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=rename(options=[NONE]).TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=rename(options=[NONE]).user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=rename.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=rename.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=rename.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=rename.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=setOwner.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=setOwner.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=setPermission.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=setPermission.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=setPermission.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=setPermission.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=setReplication.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=setReplication.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=300000.op=setTimes.TotalCount
+dfs.NNTopUserOpCounts.windowMs=300000.op=setTimes.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=*.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=*.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=*.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=*.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=*.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=*.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=create.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=create.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=create.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=create.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=delete.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=delete.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=delete.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=getfileinfo.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=getfileinfo.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=getfileinfo.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=getfileinfo.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=getfileinfo.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=getfileinfo.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=listStatus.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=listStatus.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=listStatus.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=listStatus.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=listStatus.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=listStatus.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=mkdirs.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=mkdirs.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=mkdirs.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=mkdirs.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=mkdirs.user=mapred.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=open.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=open.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=open.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=open.user=yarn.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=rename.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=rename.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=rename.user=ams.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=setOwner.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=setOwner.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=setPermission.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=setPermission.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=setPermission.user=hdfs.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=setReplication.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=setReplication.user=ambari-qa.count
+dfs.NNTopUserOpCounts.windowMs=60000.op=setTimes.TotalCount
+dfs.NNTopUserOpCounts.windowMs=60000.op=setTimes.user=ams.count
dfs.namenode.AddBlockOps
dfs.namenode.AllowSnapshotOps
+dfs.namenode.BlockOpsBatched
+dfs.namenode.BlockOpsQueued
dfs.namenode.BlockReceivedAndDeletedOps
dfs.namenode.BlockReportAvgTime
dfs.namenode.BlockReportNumOps
@@ -65,6 +241,7 @@
dfs.namenode.FilesDeleted
dfs.namenode.FilesInGetListingOps
dfs.namenode.FilesRenamed
+dfs.namenode.FilesTruncated
dfs.namenode.FsImageLoadTime
dfs.namenode.GetAdditionalDatanodeOps
dfs.namenode.GetBlockLocations
@@ -83,15 +260,19 @@
dfs.namenode.StorageBlockReportOps
dfs.namenode.SyncsAvgTime
dfs.namenode.SyncsNumOps
+dfs.namenode.TotalFileOps
dfs.namenode.TransactionsAvgTime
dfs.namenode.TransactionsBatchedInSync
dfs.namenode.TransactionsNumOps
jvm.JvmMetrics.GcCount
jvm.JvmMetrics.GcCountConcurrentMarkSweep
jvm.JvmMetrics.GcCountParNew
+jvm.JvmMetrics.GcNumInfoThresholdExceeded
+jvm.JvmMetrics.GcNumWarnThresholdExceeded
jvm.JvmMetrics.GcTimeMillis
jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep
jvm.JvmMetrics.GcTimeMillisParNew
+jvm.JvmMetrics.GcTotalExtraSleepTime
jvm.JvmMetrics.LogError
jvm.JvmMetrics.LogFatal
jvm.JvmMetrics.LogInfo
@@ -125,81 +306,91 @@
rpc.RetryCache.NameNodeRetryCache.CacheCleared
rpc.RetryCache.NameNodeRetryCache.CacheHit
rpc.RetryCache.NameNodeRetryCache.CacheUpdated
-rpc.rpc.CallQueueLength
-rpc.rpc.NumOpenConnections
-rpc.rpc.ReceivedBytes
-rpc.rpc.RpcAuthenticationFailures
-rpc.rpc.RpcAuthenticationSuccesses
-rpc.rpc.RpcAuthorizationFailures
-rpc.rpc.RpcAuthorizationSuccesses
-rpc.rpc.RpcProcessingTimeAvgTime
-rpc.rpc.RpcProcessingTimeNumOps
-rpc.rpc.RpcQueueTimeAvgTime
-rpc.rpc.RpcQueueTimeNumOps
-rpc.rpc.SentBytes
-rpcdetailed.rpcdetailed.AbandonBlockAvgTime
-rpcdetailed.rpcdetailed.AbandonBlockNumOps
-rpcdetailed.rpcdetailed.AddBlockAvgTime
-rpcdetailed.rpcdetailed.AddBlockNumOps
-rpcdetailed.rpcdetailed.BlockReceivedAndDeletedAvgTime
-rpcdetailed.rpcdetailed.BlockReceivedAndDeletedNumOps
-rpcdetailed.rpcdetailed.BlockReportAvgTime
-rpcdetailed.rpcdetailed.BlockReportNumOps
-rpcdetailed.rpcdetailed.CommitBlockSynchronizationAvgTime
-rpcdetailed.rpcdetailed.CommitBlockSynchronizationNumOps
-rpcdetailed.rpcdetailed.CompleteAvgTime
-rpcdetailed.rpcdetailed.CompleteNumOps
-rpcdetailed.rpcdetailed.CreateAvgTime
-rpcdetailed.rpcdetailed.CreateNumOps
-rpcdetailed.rpcdetailed.DeleteAvgTime
-rpcdetailed.rpcdetailed.DeleteNumOps
-rpcdetailed.rpcdetailed.FsyncAvgTime
-rpcdetailed.rpcdetailed.FsyncNumOps
-rpcdetailed.rpcdetailed.GetAdditionalDatanodeAvgTime
-rpcdetailed.rpcdetailed.GetAdditionalDatanodeNumOps
-rpcdetailed.rpcdetailed.GetBlockLocationsAvgTime
-rpcdetailed.rpcdetailed.GetBlockLocationsNumOps
-rpcdetailed.rpcdetailed.GetEditLogManifestAvgTime
-rpcdetailed.rpcdetailed.GetEditLogManifestNumOps
-rpcdetailed.rpcdetailed.GetFileInfoAvgTime
-rpcdetailed.rpcdetailed.GetFileInfoNumOps
-rpcdetailed.rpcdetailed.GetListingAvgTime
-rpcdetailed.rpcdetailed.GetListingNumOps
-rpcdetailed.rpcdetailed.GetServerDefaultsAvgTime
-rpcdetailed.rpcdetailed.GetServerDefaultsNumOps
-rpcdetailed.rpcdetailed.GetTransactionIdAvgTime
-rpcdetailed.rpcdetailed.GetTransactionIdNumOps
-rpcdetailed.rpcdetailed.IsFileClosedAvgTime
-rpcdetailed.rpcdetailed.IsFileClosedNumOps
-rpcdetailed.rpcdetailed.MkdirsAvgTime
-rpcdetailed.rpcdetailed.MkdirsNumOps
-rpcdetailed.rpcdetailed.RecoverLeaseAvgTime
-rpcdetailed.rpcdetailed.RecoverLeaseNumOps
-rpcdetailed.rpcdetailed.RegisterDatanodeAvgTime
-rpcdetailed.rpcdetailed.RegisterDatanodeNumOps
-rpcdetailed.rpcdetailed.RenameAvgTime
-rpcdetailed.rpcdetailed.RenameNumOps
-rpcdetailed.rpcdetailed.RenewLeaseAvgTime
-rpcdetailed.rpcdetailed.RenewLeaseNumOps
-rpcdetailed.rpcdetailed.ReportBadBlocksAvgTime
-rpcdetailed.rpcdetailed.ReportBadBlocksNumOps
-rpcdetailed.rpcdetailed.RollEditLogAvgTime
-rpcdetailed.rpcdetailed.RollEditLogNumOps
-rpcdetailed.rpcdetailed.SendHeartbeatAvgTime
-rpcdetailed.rpcdetailed.SendHeartbeatNumOps
-rpcdetailed.rpcdetailed.SetSafeModeAvgTime
-rpcdetailed.rpcdetailed.SetSafeModeNumOps
-rpcdetailed.rpcdetailed.SetTimesAvgTime
-rpcdetailed.rpcdetailed.SetTimesNumOps
-rpcdetailed.rpcdetailed.UpdateBlockForPipelineAvgTime
-rpcdetailed.rpcdetailed.UpdateBlockForPipelineNumOps
-rpcdetailed.rpcdetailed.UpdatePipelineAvgTime
-rpcdetailed.rpcdetailed.UpdatePipelineNumOps
-rpcdetailed.rpcdetailed.VersionRequestAvgTime
-rpcdetailed.rpcdetailed.VersionRequestNumOps
+rpc.rpc.client.CallQueueLength
+rpc.rpc.client.NumOpenConnections
+rpc.rpc.client.ReceivedBytes
+rpc.rpc.client.RpcAuthenticationFailures
+rpc.rpc.client.RpcAuthenticationSuccesses
+rpc.rpc.client.RpcAuthorizationFailures
+rpc.rpc.client.RpcAuthorizationSuccesses
+rpc.rpc.client.RpcClientBackoff
+rpc.rpc.client.RpcProcessingTimeAvgTime
+rpc.rpc.client.RpcProcessingTimeNumOps
+rpc.rpc.client.RpcQueueTimeAvgTime
+rpc.rpc.client.RpcQueueTimeNumOps
+rpc.rpc.client.RpcSlowCalls
+rpc.rpc.client.SentBytes
+rpcdetailed.rpcdetailed.client.AddBlockAvgTime
+rpcdetailed.rpcdetailed.client.AddBlockNumOps
+rpcdetailed.rpcdetailed.client.BlockReceivedAndDeletedAvgTime
+rpcdetailed.rpcdetailed.client.BlockReceivedAndDeletedNumOps
+rpcdetailed.rpcdetailed.client.BlockReportAvgTime
+rpcdetailed.rpcdetailed.client.BlockReportNumOps
+rpcdetailed.rpcdetailed.client.CheckAccessAvgTime
+rpcdetailed.rpcdetailed.client.CheckAccessNumOps
+rpcdetailed.rpcdetailed.client.CommitBlockSynchronizationAvgTime
+rpcdetailed.rpcdetailed.client.CommitBlockSynchronizationNumOps
+rpcdetailed.rpcdetailed.client.CompleteAvgTime
+rpcdetailed.rpcdetailed.client.CompleteNumOps
+rpcdetailed.rpcdetailed.client.CreateAvgTime
+rpcdetailed.rpcdetailed.client.CreateNumOps
+rpcdetailed.rpcdetailed.client.DeleteAvgTime
+rpcdetailed.rpcdetailed.client.DeleteNumOps
+rpcdetailed.rpcdetailed.client.FsyncAvgTime
+rpcdetailed.rpcdetailed.client.FsyncNumOps
+rpcdetailed.rpcdetailed.client.GetBlockLocationsAvgTime
+rpcdetailed.rpcdetailed.client.GetBlockLocationsNumOps
+rpcdetailed.rpcdetailed.client.GetEditLogManifestAvgTime
+rpcdetailed.rpcdetailed.client.GetEditLogManifestNumOps
+rpcdetailed.rpcdetailed.client.GetFileInfoAvgTime
+rpcdetailed.rpcdetailed.client.GetFileInfoNumOps
+rpcdetailed.rpcdetailed.client.GetListingAvgTime
+rpcdetailed.rpcdetailed.client.GetListingNumOps
+rpcdetailed.rpcdetailed.client.GetServerDefaultsAvgTime
+rpcdetailed.rpcdetailed.client.GetServerDefaultsNumOps
+rpcdetailed.rpcdetailed.client.GetTransactionIdAvgTime
+rpcdetailed.rpcdetailed.client.GetTransactionIdNumOps
+rpcdetailed.rpcdetailed.client.IsRollingUpgradeAvgTime
+rpcdetailed.rpcdetailed.client.IsRollingUpgradeNumOps
+rpcdetailed.rpcdetailed.client.ListEncryptionZonesAvgTime
+rpcdetailed.rpcdetailed.client.ListEncryptionZonesNumOps
+rpcdetailed.rpcdetailed.client.MkdirsAvgTime
+rpcdetailed.rpcdetailed.client.MkdirsNumOps
+rpcdetailed.rpcdetailed.client.PathIsNotEmptyDirectoryExceptionAvgTime
+rpcdetailed.rpcdetailed.client.PathIsNotEmptyDirectoryExceptionNumOps
+rpcdetailed.rpcdetailed.client.RecoverLeaseAvgTime
+rpcdetailed.rpcdetailed.client.RecoverLeaseNumOps
+rpcdetailed.rpcdetailed.client.RegisterDatanodeAvgTime
+rpcdetailed.rpcdetailed.client.RegisterDatanodeNumOps
+rpcdetailed.rpcdetailed.client.Rename2AvgTime
+rpcdetailed.rpcdetailed.client.Rename2NumOps
+rpcdetailed.rpcdetailed.client.RenameAvgTime
+rpcdetailed.rpcdetailed.client.RenameNumOps
+rpcdetailed.rpcdetailed.client.RenewLeaseAvgTime
+rpcdetailed.rpcdetailed.client.RenewLeaseNumOps
+rpcdetailed.rpcdetailed.client.RetriableExceptionAvgTime
+rpcdetailed.rpcdetailed.client.RetriableExceptionNumOps
+rpcdetailed.rpcdetailed.client.RollEditLogAvgTime
+rpcdetailed.rpcdetailed.client.RollEditLogNumOps
+rpcdetailed.rpcdetailed.client.SafeModeExceptionAvgTime
+rpcdetailed.rpcdetailed.client.SafeModeExceptionNumOps
+rpcdetailed.rpcdetailed.client.SendHeartbeatAvgTime
+rpcdetailed.rpcdetailed.client.SendHeartbeatNumOps
+rpcdetailed.rpcdetailed.client.SetPermissionAvgTime
+rpcdetailed.rpcdetailed.client.SetPermissionNumOps
+rpcdetailed.rpcdetailed.client.SetReplicationAvgTime
+rpcdetailed.rpcdetailed.client.SetReplicationNumOps
+rpcdetailed.rpcdetailed.client.SetSafeModeAvgTime
+rpcdetailed.rpcdetailed.client.SetSafeModeNumOps
+rpcdetailed.rpcdetailed.client.SetTimesAvgTime
+rpcdetailed.rpcdetailed.client.SetTimesNumOps
+rpcdetailed.rpcdetailed.client.VersionRequestAvgTime
+rpcdetailed.rpcdetailed.client.VersionRequestNumOps
ugi.UgiMetrics.GetGroupsAvgTime
ugi.UgiMetrics.GetGroupsNumOps
ugi.UgiMetrics.LoginFailureAvgTime
ugi.UgiMetrics.LoginFailureNumOps
ugi.UgiMetrics.LoginSuccessAvgTime
ugi.UgiMetrics.LoginSuccessNumOps
+ugi.UgiMetrics.RenewalFailures
+ugi.UgiMetrics.RenewalFailuresTotal
\ No newline at end of file
diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/NODEMANAGER.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/NODEMANAGER.dat
index 239b3d4..e6c699e 100644
--- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/NODEMANAGER.dat
+++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/NODEMANAGER.dat
@@ -50,6 +50,14 @@
rpc.rpc.RpcQueueTimeAvgTime
rpc.rpc.RpcQueueTimeNumOps
rpc.rpc.SentBytes
+rpcdetailed.rpcdetailed.GetContainerStatusesAvgTime
+rpcdetailed.rpcdetailed.GetContainerStatusesNumOps
+rpcdetailed.rpcdetailed.HeartbeatAvgTime
+rpcdetailed.rpcdetailed.HeartbeatNumOps
+rpcdetailed.rpcdetailed.StartContainersAvgTime
+rpcdetailed.rpcdetailed.StartContainersNumOps
+rpcdetailed.rpcdetailed.StopContainersAvgTime
+rpcdetailed.rpcdetailed.StopContainersNumOps
ugi.UgiMetrics.GetGroupsAvgTime
ugi.UgiMetrics.GetGroupsNumOps
ugi.UgiMetrics.LoginFailureAvgTime
@@ -58,10 +66,18 @@
ugi.UgiMetrics.LoginSuccessNumOps
yarn.NodeManagerMetrics.AllocatedContainers
yarn.NodeManagerMetrics.AllocatedGB
+yarn.NodeManagerMetrics.AllocatedVCores
yarn.NodeManagerMetrics.AvailableGB
+yarn.NodeManagerMetrics.AvailableVCores
+yarn.NodeManagerMetrics.BadLocalDirs
+yarn.NodeManagerMetrics.BadLogDirs
+yarn.NodeManagerMetrics.ContainerLaunchDurationAvgTime
+yarn.NodeManagerMetrics.ContainerLaunchDurationNumOps
yarn.NodeManagerMetrics.ContainersCompleted
yarn.NodeManagerMetrics.ContainersFailed
yarn.NodeManagerMetrics.ContainersIniting
yarn.NodeManagerMetrics.ContainersKilled
yarn.NodeManagerMetrics.ContainersLaunched
yarn.NodeManagerMetrics.ContainersRunning
+yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc
+yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc
diff --git a/ambari-metrics-timelineservice/src/main/resources/metrics_def/RESOURCEMANAGER.dat b/ambari-metrics-timelineservice/src/main/resources/metrics_def/RESOURCEMANAGER.dat
index ec698db..2e45c22 100644
--- a/ambari-metrics-timelineservice/src/main/resources/metrics_def/RESOURCEMANAGER.dat
+++ b/ambari-metrics-timelineservice/src/main/resources/metrics_def/RESOURCEMANAGER.dat
@@ -1,88 +1,159 @@
-jvm.JvmMetrics.GcCount
-jvm.JvmMetrics.GcCountPS MarkSweep
-jvm.JvmMetrics.GcCountPS Scavenge
-jvm.JvmMetrics.GcTimeMillis
-jvm.JvmMetrics.GcTimeMillisPS MarkSweep
-jvm.JvmMetrics.GcTimeMillisPS Scavenge
-jvm.JvmMetrics.LogError
-jvm.JvmMetrics.LogFatal
-jvm.JvmMetrics.LogInfo
-jvm.JvmMetrics.LogWarn
-jvm.JvmMetrics.MemHeapCommittedM
-jvm.JvmMetrics.MemHeapMaxM
-jvm.JvmMetrics.MemHeapUsedM
-jvm.JvmMetrics.MemMaxM
-jvm.JvmMetrics.MemNonHeapCommittedM
-jvm.JvmMetrics.MemNonHeapMaxM
-jvm.JvmMetrics.MemNonHeapUsedM
-jvm.JvmMetrics.ThreadsBlocked
-jvm.JvmMetrics.ThreadsNew
-jvm.JvmMetrics.ThreadsRunnable
-jvm.JvmMetrics.ThreadsTerminated
-jvm.JvmMetrics.ThreadsTimedWaiting
-jvm.JvmMetrics.ThreadsWaiting
-metricssystem.MetricsSystem.DroppedPubAll
-metricssystem.MetricsSystem.NumActiveSinks
-metricssystem.MetricsSystem.NumActiveSources
-metricssystem.MetricsSystem.NumAllSinks
-metricssystem.MetricsSystem.NumAllSources
-metricssystem.MetricsSystem.PublishAvgTime
-metricssystem.MetricsSystem.PublishNumOps
-metricssystem.MetricsSystem.Sink_timelineAvgTime
-metricssystem.MetricsSystem.Sink_timelineDropped
-metricssystem.MetricsSystem.Sink_timelineNumOps
-metricssystem.MetricsSystem.Sink_timelineQsize
-metricssystem.MetricsSystem.SnapshotAvgTime
-metricssystem.MetricsSystem.SnapshotNumOps
-rpc.rpc.CallQueueLength
-rpc.rpc.NumOpenConnections
-rpc.rpc.ReceivedBytes
-rpc.rpc.RpcAuthenticationFailures
-rpc.rpc.RpcAuthenticationSuccesses
-rpc.rpc.RpcAuthorizationFailures
-rpc.rpc.RpcAuthorizationSuccesses
-rpc.rpc.RpcProcessingTimeAvgTime
-rpc.rpc.RpcProcessingTimeNumOps
-rpc.rpc.RpcQueueTimeAvgTime
-rpc.rpc.RpcQueueTimeNumOps
-rpc.rpc.SentBytes
-rpcdetailed.rpcdetailed.NodeHeartbeatAvgTime
-rpcdetailed.rpcdetailed.NodeHeartbeatNumOps
-rpcdetailed.rpcdetailed.RegisterNodeManagerAvgTime
-rpcdetailed.rpcdetailed.RegisterNodeManagerNumOps
-ugi.UgiMetrics.GetGroupsAvgTime
-ugi.UgiMetrics.GetGroupsNumOps
-ugi.UgiMetrics.LoginFailureAvgTime
-ugi.UgiMetrics.LoginFailureNumOps
-ugi.UgiMetrics.LoginSuccessAvgTime
-ugi.UgiMetrics.LoginSuccessNumOps
-yarn.ClusterMetrics.NumActiveNMs
-yarn.ClusterMetrics.NumDecommissionedNMs
-yarn.ClusterMetrics.NumLostNMs
-yarn.ClusterMetrics.NumRebootedNMs
-yarn.ClusterMetrics.NumUnhealthyNMs
-yarn.QueueMetrics.ActiveApplications
-yarn.QueueMetrics.ActiveUsers
-yarn.QueueMetrics.AggregateContainersAllocated
-yarn.QueueMetrics.AggregateContainersReleased
-yarn.QueueMetrics.AllocatedContainers
-yarn.QueueMetrics.AllocatedMB
-yarn.QueueMetrics.AllocatedVCores
-yarn.QueueMetrics.AppsCompleted
-yarn.QueueMetrics.AppsFailed
-yarn.QueueMetrics.AppsKilled
-yarn.QueueMetrics.AppsPending
-yarn.QueueMetrics.AppsRunning
-yarn.QueueMetrics.AppsSubmitted
-yarn.QueueMetrics.AvailableMB
-yarn.QueueMetrics.AvailableVCores
-yarn.QueueMetrics.PendingContainers
-yarn.QueueMetrics.PendingMB
-yarn.QueueMetrics.PendingVCores
-yarn.QueueMetrics.ReservedContainers
-yarn.QueueMetrics.ReservedMB
-yarn.QueueMetrics.ReservedVCores
-yarn.QueueMetrics.running_0
-yarn.QueueMetrics.running_1440
-yarn.QueueMetrics.running_300
-yarn.QueueMetrics.running_60
+vm.JvmMetrics.GcCount
+jvm.JvmMetrics.GcCountPS MarkSweep
+jvm.JvmMetrics.GcCountPS Scavenge
+jvm.JvmMetrics.GcTimeMillis
+jvm.JvmMetrics.GcTimeMillisPS MarkSweep
+jvm.JvmMetrics.GcTimeMillisPS Scavenge
+jvm.JvmMetrics.LogError
+jvm.JvmMetrics.LogFatal
+jvm.JvmMetrics.LogInfo
+jvm.JvmMetrics.LogWarn
+jvm.JvmMetrics.MemHeapCommittedM
+jvm.JvmMetrics.MemHeapMaxM
+jvm.JvmMetrics.MemHeapUsedM
+jvm.JvmMetrics.MemMaxM
+jvm.JvmMetrics.MemNonHeapCommittedM
+jvm.JvmMetrics.MemNonHeapMaxM
+jvm.JvmMetrics.MemNonHeapUsedM
+jvm.JvmMetrics.ThreadsBlocked
+jvm.JvmMetrics.ThreadsNew
+jvm.JvmMetrics.ThreadsRunnable
+jvm.JvmMetrics.ThreadsTerminated
+jvm.JvmMetrics.ThreadsTimedWaiting
+jvm.JvmMetrics.ThreadsWaiting
+metricssystem.MetricsSystem.DroppedPubAll
+metricssystem.MetricsSystem.NumActiveSinks
+metricssystem.MetricsSystem.NumActiveSources
+metricssystem.MetricsSystem.NumAllSinks
+metricssystem.MetricsSystem.NumAllSources
+metricssystem.MetricsSystem.PublishAvgTime
+metricssystem.MetricsSystem.PublishNumOps
+metricssystem.MetricsSystem.Sink_timelineAvgTime
+metricssystem.MetricsSystem.Sink_timelineDropped
+metricssystem.MetricsSystem.Sink_timelineNumOps
+metricssystem.MetricsSystem.Sink_timelineQsize
+metricssystem.MetricsSystem.SnapshotAvgTime
+metricssystem.MetricsSystem.SnapshotNumOps
+rpc.rpc.CallQueueLength
+rpc.rpc.NumOpenConnections
+rpc.rpc.ReceivedBytes
+rpc.rpc.RpcAuthenticationFailures
+rpc.rpc.RpcAuthenticationSuccesses
+rpc.rpc.RpcAuthorizationFailures
+rpc.rpc.RpcAuthorizationSuccesses
+rpc.rpc.RpcClientBackoff
+rpc.rpc.RpcProcessingTimeAvgTime
+rpc.rpc.RpcProcessingTimeNumOps
+rpc.rpc.RpcQueueTimeAvgTime
+rpc.rpc.RpcQueueTimeNumOps
+rpc.rpc.RpcSlowCalls
+rpc.rpc.SentBytes
+rpcdetailed.rpcdetailed.AllocateAvgTime
+rpcdetailed.rpcdetailed.AllocateNumOps
+rpcdetailed.rpcdetailed.FinishApplicationMasterAvgTime
+rpcdetailed.rpcdetailed.FinishApplicationMasterNumOps
+rpcdetailed.rpcdetailed.GetApplicationReportAvgTime
+rpcdetailed.rpcdetailed.GetApplicationReportNumOps
+rpcdetailed.rpcdetailed.GetClusterMetricsAvgTime
+rpcdetailed.rpcdetailed.GetClusterMetricsNumOps
+rpcdetailed.rpcdetailed.GetClusterNodesAvgTime
+rpcdetailed.rpcdetailed.GetClusterNodesNumOps
+rpcdetailed.rpcdetailed.GetNewApplicationAvgTime
+rpcdetailed.rpcdetailed.GetNewApplicationNumOps
+rpcdetailed.rpcdetailed.GetQueueInfoAvgTime
+rpcdetailed.rpcdetailed.GetQueueInfoNumOps
+rpcdetailed.rpcdetailed.GetQueueUserAclsAvgTime
+rpcdetailed.rpcdetailed.GetQueueUserAclsNumOps
+rpcdetailed.rpcdetailed.NodeHeartbeatAvgTime
+rpcdetailed.rpcdetailed.NodeHeartbeatNumOps
+rpcdetailed.rpcdetailed.RegisterApplicationMasterAvgTime
+rpcdetailed.rpcdetailed.RegisterApplicationMasterNumOps
+rpcdetailed.rpcdetailed.RegisterNodeManagerAvgTime
+rpcdetailed.rpcdetailed.RegisterNodeManagerNumOps
+rpcdetailed.rpcdetailed.SubmitApplicationAvgTime
+rpcdetailed.rpcdetailed.SubmitApplicationNumOps
+ugi.UgiMetrics.GetGroupsAvgTime
+ugi.UgiMetrics.GetGroupsNumOps
+ugi.UgiMetrics.LoginFailureAvgTime
+ugi.UgiMetrics.LoginFailureNumOps
+ugi.UgiMetrics.LoginSuccessAvgTime
+ugi.UgiMetrics.LoginSuccessNumOps
+ugi.UgiMetrics.RenewalFailures
+ugi.UgiMetrics.RenewalFailuresTotal
+yarn.ClusterMetrics.AMLaunchDelayAvgTime
+yarn.ClusterMetrics.AMLaunchDelayNumOps
+yarn.ClusterMetrics.AMRegisterDelayAvgTime
+yarn.ClusterMetrics.AMRegisterDelayNumOps
+yarn.ClusterMetrics.NumActiveNMs
+yarn.ClusterMetrics.NumDecommissionedNMs
+yarn.ClusterMetrics.NumLostNMs
+yarn.ClusterMetrics.NumRebootedNMs
+yarn.ClusterMetrics.NumUnhealthyNMs
+yarn.QueueMetrics.Queue=root.AMResourceLimitMB
+yarn.QueueMetrics.Queue=root.AMResourceLimitVCores
+yarn.QueueMetrics.Queue=root.ActiveApplications
+yarn.QueueMetrics.Queue=root.ActiveUsers
+yarn.QueueMetrics.Queue=root.AggregateContainersAllocated
+yarn.QueueMetrics.Queue=root.AggregateContainersReleased
+yarn.QueueMetrics.Queue=root.AggregateNodeLocalContainersAllocated
+yarn.QueueMetrics.Queue=root.AggregateOffSwitchContainersAllocated
+yarn.QueueMetrics.Queue=root.AggregateRackLocalContainersAllocated
+yarn.QueueMetrics.Queue=root.AllocatedContainers
+yarn.QueueMetrics.Queue=root.AllocatedMB
+yarn.QueueMetrics.Queue=root.AllocatedVCores
+yarn.QueueMetrics.Queue=root.AppAttemptFirstContainerAllocationDelayAvgTime
+yarn.QueueMetrics.Queue=root.AppAttemptFirstContainerAllocationDelayNumOps
+yarn.QueueMetrics.Queue=root.AppsCompleted
+yarn.QueueMetrics.Queue=root.AppsFailed
+yarn.QueueMetrics.Queue=root.AppsKilled
+yarn.QueueMetrics.Queue=root.AppsPending
+yarn.QueueMetrics.Queue=root.AppsRunning
+yarn.QueueMetrics.Queue=root.AppsSubmitted
+yarn.QueueMetrics.Queue=root.AvailableMB
+yarn.QueueMetrics.Queue=root.AvailableVCores
+yarn.QueueMetrics.Queue=root.PendingContainers
+yarn.QueueMetrics.Queue=root.PendingMB
+yarn.QueueMetrics.Queue=root.PendingVCores
+yarn.QueueMetrics.Queue=root.ReservedContainers
+yarn.QueueMetrics.Queue=root.ReservedMB
+yarn.QueueMetrics.Queue=root.ReservedVCores
+yarn.QueueMetrics.Queue=root.UsedAMResourceMB
+yarn.QueueMetrics.Queue=root.UsedAMResourceVCores
+yarn.QueueMetrics.Queue=root.default.AMResourceLimitMB
+yarn.QueueMetrics.Queue=root.default.AMResourceLimitVCores
+yarn.QueueMetrics.Queue=root.default.ActiveApplications
+yarn.QueueMetrics.Queue=root.default.ActiveUsers
+yarn.QueueMetrics.Queue=root.default.AggregateContainersAllocated
+yarn.QueueMetrics.Queue=root.default.AggregateContainersReleased
+yarn.QueueMetrics.Queue=root.default.AggregateNodeLocalContainersAllocated
+yarn.QueueMetrics.Queue=root.default.AggregateOffSwitchContainersAllocated
+yarn.QueueMetrics.Queue=root.default.AggregateRackLocalContainersAllocated
+yarn.QueueMetrics.Queue=root.default.AllocatedContainers
+yarn.QueueMetrics.Queue=root.default.AllocatedMB
+yarn.QueueMetrics.Queue=root.default.AllocatedVCores
+yarn.QueueMetrics.Queue=root.default.AppAttemptFirstContainerAllocationDelayAvgTime
+yarn.QueueMetrics.Queue=root.default.AppAttemptFirstContainerAllocationDelayNumOps
+yarn.QueueMetrics.Queue=root.default.AppsCompleted
+yarn.QueueMetrics.Queue=root.default.AppsFailed
+yarn.QueueMetrics.Queue=root.default.AppsKilled
+yarn.QueueMetrics.Queue=root.default.AppsPending
+yarn.QueueMetrics.Queue=root.default.AppsRunning
+yarn.QueueMetrics.Queue=root.default.AppsSubmitted
+yarn.QueueMetrics.Queue=root.default.AvailableMB
+yarn.QueueMetrics.Queue=root.default.AvailableVCores
+yarn.QueueMetrics.Queue=root.default.PendingContainers
+yarn.QueueMetrics.Queue=root.default.PendingMB
+yarn.QueueMetrics.Queue=root.default.PendingVCores
+yarn.QueueMetrics.Queue=root.default.ReservedContainers
+yarn.QueueMetrics.Queue=root.default.ReservedMB
+yarn.QueueMetrics.Queue=root.default.ReservedVCores
+yarn.QueueMetrics.Queue=root.default.UsedAMResourceMB
+yarn.QueueMetrics.Queue=root.default.UsedAMResourceVCores
+yarn.QueueMetrics.Queue=root.default.running_0
+yarn.QueueMetrics.Queue=root.default.running_1440
+yarn.QueueMetrics.Queue=root.default.running_300
+yarn.QueueMetrics.Queue=root.default.running_60
+yarn.QueueMetrics.Queue=root.running_0
+yarn.QueueMetrics.Queue=root.running_1440
+yarn.QueueMetrics.Queue=root.running_300
+yarn.QueueMetrics.Queue=root.running_60
\ No newline at end of file
diff --git a/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java b/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java
index c34ac20..7f168f2 100644
--- a/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java
+++ b/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/jmetertest/AMSJMeterLoadTest.java
@@ -45,8 +45,10 @@
private List<AppGetMetric> appGetMetrics;
private Properties amsJmeterProperties = null;
- public AMSJMeterLoadTest(String testType, String userDefinedPropertiesFile) {
+ public AMSJMeterLoadTest(Map<String, String> args) {
+ String testType = args.get("type");
+ String userDefinedPropertiesFile = args.get("amsJmeterPropertiesFile");
if (null == userDefinedPropertiesFile || userDefinedPropertiesFile.isEmpty()) {
this.amsJmeterProperties = readProperties(PROPERTIES_FILE);
} else {
@@ -60,12 +62,14 @@
this.runTest(numInstances);
} else { //PUT Metrics simulator
Map<String, String> mapArgs = new HashMap<String, String>();
- mapArgs.put("hostName", amsJmeterProperties.getProperty("host-prefix"));
- mapArgs.put("numberOfHosts", amsJmeterProperties.getProperty("num-hosts"));
- mapArgs.put("metricsHostName", amsJmeterProperties.getProperty("ams-host-port"));
- mapArgs.put("collectInterval", amsJmeterProperties.getProperty("collection-interval"));
- mapArgs.put("sendInterval", amsJmeterProperties.getProperty("send-interval"));
- mapArgs.put("master", amsJmeterProperties.getProperty("create-master"));
+ mapArgs.put("hostName", (args.get("host-prefix") != null) ? args.get("host-prefix") : amsJmeterProperties.getProperty("host-prefix"));
+ mapArgs.put("minHostIndex", (args.get("min-host-index") != null) ? args.get("min-host-index") : amsJmeterProperties.getProperty("min-host-index"));
+ mapArgs.put("numberOfHosts", (args.get("num-hosts") != null) ? args.get("num-hosts") : amsJmeterProperties.getProperty("num-hosts"));
+ mapArgs.put("metricsHostName", (args.get("ams-host-port") != null) ? args.get("ams-host-port") : amsJmeterProperties.getProperty("ams-host-port"));
+ mapArgs.put("collectInterval", (args.get("collection-interval") != null) ? args.get("collection-interval") : amsJmeterProperties.getProperty("collection-interval"));
+ mapArgs.put("sendInterval", (args.get("send-interval") != null) ? args.get("send-interval") : amsJmeterProperties.getProperty("send-interval"));
+ mapArgs.put("master", (args.get("create-master") != null) ? args.get("create-master") : amsJmeterProperties.getProperty("create-master"));
+ System.out.println("AMS Load Simulation Parameters : " + mapArgs);
MetricsLoadSimulator.startTest(mapArgs);
}
}
@@ -165,9 +169,7 @@
*/
public static void main(String[] args) {
Map<String, String> mapArgs = parseArgs(args);
- String testType = mapArgs.get("type");
- String amsJmeterPropertiesFile = mapArgs.get("amsJmeterPropertiesFile");
- new AMSJMeterLoadTest(testType, amsJmeterPropertiesFile);
+ new AMSJMeterLoadTest(mapArgs);
}
private static Map<String, String> parseArgs(String[] args) {
@@ -178,14 +180,7 @@
} else {
for (int i = 0; i < args.length; i += 2) {
String arg = args[i];
- if (arg.equals("-t")) {
- mapProps.put("type", args[i + 1]);
- } else if (arg.equals("-p")) {
- mapProps.put("amsJmeterPropertiesFile", args[i + 1]);
- } else {
- printUsage();
- throw new IllegalArgumentException("Unexpected argument, See usage message.");
- }
+ mapProps.put(arg.substring(1), args[i+1]);
}
}
return mapProps;
@@ -194,7 +189,8 @@
public static void printUsage() {
System.err.println("Usage: java AMSJmeterLoadTest [OPTIONS]");
System.err.println("Options: ");
- System.err.println("[-t type (S=>Sink/U=>UI)] [-p amsJmeterPropertiesFile (Optional)]");
+ System.err.println("[--t type (S=>Sink/U=>UI)] [-ams-host-port localhost:6188] [-min-host-index 2] [-host-prefix TestHost.] [-num-hosts 2] " +
+ "[-create-master true] [-collection-interval 10000 ] [-send-interval 60000 ] [-p amsJmeterPropertiesFile (Optional)]");
}
}
diff --git a/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java b/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java
index 73c66fe..81da5c8 100644
--- a/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java
+++ b/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java
@@ -147,4 +147,74 @@
Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
}
+ @Test
+ public void testHybridFilter() throws Exception {
+
+ // Whitelist Apps - namenode, nimbus
+ // Blacklist Apps - datanode, kafka_broker
+ // Accept ams-hbase whitelisting.
+ // Reject non whitelisted metrics from non whitelisted Apps (Say hbase)
+
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+
+ Configuration metricsConf = new Configuration();
+ metricsConf.set("timeline.metrics.apps.whitelist", "namenode,nimbus");
+ metricsConf.set("timeline.metrics.apps.blacklist", "datanode,kafka_broker");
+ URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
+ metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+
+ Set<String> whitelist = new HashSet();
+ whitelist.add("regionserver.Server.Delete_99th_percentile");
+ whitelist.add("regionserver.Server.Delete_max");
+ whitelist.add("regionserver.Server.Delete_mean");
+ expect(configuration.getAmshbaseWhitelist()).andReturn(whitelist).once();
+
+ replay(configuration);
+
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
+
+ TimelineMetric timelineMetric = new TimelineMetric();
+
+ //Test App Whitelisting
+ timelineMetric.setMetricName("metric.a.b.c");
+ timelineMetric.setAppId("namenode");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("metric.d.e.f");
+ timelineMetric.setAppId("nimbus");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ //Test App Blacklisting
+ timelineMetric.setMetricName("metric.d.e.f");
+ timelineMetric.setAppId("datanode");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("metric.d.e.f");
+ timelineMetric.setAppId("kafka_broker");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+
+ //Test ams-hbase Whitelisting
+ timelineMetric.setMetricName("regionserver.Server.Delete_max");
+ timelineMetric.setAppId("ams-hbase");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("regionserver.Server.Delete_min3333");
+ timelineMetric.setAppId("ams-hbase");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("regionserver.Server.Delete_mean");
+ timelineMetric.setAppId("ams-hbase");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ //Test Metric Whitelisting
+ timelineMetric.setMetricName("regionserver.WAL.SyncTime_max");
+ timelineMetric.setAppId("hbase");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("regionserver.WAL.metric.not.needed");
+ timelineMetric.setAppId("hbase");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+ }
}
diff --git a/ambari-metrics-timelineservice/src/test/resources/loadsimulator/README b/ambari-metrics-timelineservice/src/test/resources/loadsimulator/README
index 39e5365..eec3e5d 100644
--- a/ambari-metrics-timelineservice/src/test/resources/loadsimulator/README
+++ b/ambari-metrics-timelineservice/src/test/resources/loadsimulator/README
@@ -41,7 +41,7 @@
2. Build the ambari-metrics-timelineservice jar.
3. Invoke the test using the command as follows.
- java -cp lib/*:ambari-metrics-timelineservice-<version>.jar org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmeter.AMSJMeterLoadTest -t <S/U>
+ java -cp lib/*:ambari-metrics-timelineservice-<version>.jar org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest.jmetertest.AMSJMeterLoadTest -t <S/U>
lib/* -> All the dependency JARs generated for the ambari-metrics-timelineservice JAR.
-t option => S-Sink simulator or U-UI simulator
diff --git a/ambari-metrics-timelineservice/src/test/resources/loadsimulator/ams-jmeter.properties b/ambari-metrics-timelineservice/src/test/resources/loadsimulator/ams-jmeter.properties
index 2c44d89..04293df 100644
--- a/ambari-metrics-timelineservice/src/test/resources/loadsimulator/ams-jmeter.properties
+++ b/ambari-metrics-timelineservice/src/test/resources/loadsimulator/ams-jmeter.properties
@@ -17,13 +17,13 @@
# limitations under the License.
#AMS information
-ams-host-port=c6402.ambari.apache.org:6188
+ams-host-port=localhost:6188
ams-path=/ws/v1/timeline/metrics
# Host Information, using the host-prefix, num-hosts and <MASTER>-host
# Hosts in this case will be TestHost0, TestHost1
# For example NAMENODE, NIMBUS,HBASE MASTER,RESOURCEMANAGER host will be TestHost0
-host-prefix=TestHost
+host-prefix=TestHost.
host-suffix=
min-host-index=0
num-hosts=2
@@ -33,8 +33,8 @@
RESOURCEMANAGER-host=0
# PUT Metric / Sinks config
-collection-interval=1000
-send-interval=3000
+collection-interval=10000
+send-interval=60000
create-master=true
# GET Metric / Client Apps config
diff --git a/ambari-metrics-timelineservice/src/test/resources/test_data/metric_whitelist.dat b/ambari-metrics-timelineservice/src/test/resources/test_data/metric_whitelist.dat
index 9f5e25c..2c36031 100644
--- a/ambari-metrics-timelineservice/src/test/resources/test_data/metric_whitelist.dat
+++ b/ambari-metrics-timelineservice/src/test/resources/test_data/metric_whitelist.dat
@@ -2,5 +2,7 @@
cpu_user
mem_cached
mem_free
+regionserver.WAL.SyncTime_max
+regionserver.WAL.SyncTime_mean
._p_dfs.FSNamesystem.*
._p_jvm.JvmMetrics*
\ No newline at end of file
