AMBARI-20777 : AMS changes to use instanceId for cluster based segregation of data. (Addendum patch) (avijayan)
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index fa095a0..c242a2f 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -19,6 +19,7 @@
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -51,6 +52,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -388,8 +390,50 @@
}
@Override
- public Map<String, Set<String>> getInstanceHostsMetadata() throws SQLException, IOException {
- return metricMetadataManager.getHostedInstanceCache();
+ public Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId)
+ throws SQLException, IOException {
+
+ Map<String, Set<String>> hostedApps = metricMetadataManager.getHostedAppsCache();
+ Map<String, Set<String>> instanceHosts = metricMetadataManager.getHostedInstanceCache();
+ Map<String, Map<String, Set<String>>> instanceAppHosts = new HashMap<>();
+
+ if (MapUtils.isEmpty(instanceHosts)) {
+ Map<String, Set<String>> appHostMap = new HashMap<String, Set<String>>();
+ for (String host : hostedApps.keySet()) {
+ for (String app : hostedApps.get(host)) {
+ if (!appHostMap.containsKey(app)) {
+ appHostMap.put(app, new HashSet<String>());
+ }
+ appHostMap.get(app).add(host);
+ }
+ }
+ instanceAppHosts.put("", appHostMap);
+ } else {
+ for (String instance : instanceHosts.keySet()) {
+
+ if (StringUtils.isNotEmpty(instanceId) && !instance.equals(instanceId)) {
+ continue;
+ }
+ Map<String, Set<String>> appHostMap = new HashMap<String, Set<String>>();
+ instanceAppHosts.put(instance, appHostMap);
+
+ Set<String> hostsWithInstance = instanceHosts.get(instance);
+ for (String host : hostsWithInstance) {
+ for (String app : hostedApps.get(host)) {
+ if (StringUtils.isNotEmpty(appId) && !app.equals(appId)) {
+ continue;
+ }
+
+ if (!appHostMap.containsKey(app)) {
+ appHostMap.put(app, new HashSet<String>());
+ }
+ appHostMap.get(app).add(host);
+ }
+ }
+ }
+ }
+
+ return instanceAppHosts;
}
@Override
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index 65bbc4c..fb369e8 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -787,7 +787,9 @@
metadataManager.putIfModifiedHostedAppsMetadata(
tm.getHostName(), tm.getAppId());
- metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(), tm.getHostName());
+ if (!tm.getAppId().equals("FLUME_HANDLER")) {
+ metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(), tm.getHostName());
+ }
}
if (!acceptMetric) {
iterator.remove();
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java
index 121a8ae..bde09cb 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java
@@ -94,7 +94,7 @@
* @throws SQLException
* @throws IOException
*/
- Map<String, Set<String>> getInstanceHostsMetadata() throws SQLException, IOException;
+ Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) throws SQLException, IOException;
/**
* Return a list of known live collector nodes
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
index 6278c59..9da921a 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
@@ -415,14 +415,16 @@
@GET
@Path("/metrics/instances")
@Produces({ MediaType.APPLICATION_JSON })
- public Map<String, Set<String>> getClusterHostsMetadata(
+ public Map<String, Map<String, Set<String>>> getClusterHostsMetadata(
@Context HttpServletRequest req,
- @Context HttpServletResponse res
+ @Context HttpServletResponse res,
+ @QueryParam("appId") String appId,
+ @QueryParam("instanceId") String instanceId
) {
init(res);
try {
- return timelineMetricStore.getInstanceHostsMetadata();
+ return timelineMetricStore.getInstanceHostsMetadata(instanceId, appId);
} catch (Exception e) {
throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
}
diff --git a/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
index b40481d..f00906e 100644
--- a/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
+++ b/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
@@ -97,7 +97,7 @@
}
@Override
- public Map<String, Set<String>> getInstanceHostsMetadata() throws SQLException, IOException {
+ public Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) throws SQLException, IOException {
return Collections.emptyMap();
}
