Merge pull request #9 from ishanbha/AMBARI-24884-master
[AMBARI-24884] AMS Grafana query editor panel does not work in upgraded version.
diff --git a/ambari-metrics-common/pom.xml b/ambari-metrics-common/pom.xml
index 1c092dc..cb8333c 100644
--- a/ambari-metrics-common/pom.xml
+++ b/ambari-metrics-common/pom.xml
@@ -162,7 +162,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>14.0.1</version>
+ <version>18.0</version>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
diff --git a/ambari-metrics-flume-sink/pom.xml b/ambari-metrics-flume-sink/pom.xml
index 72ebb27..7b4f4fa 100644
--- a/ambari-metrics-flume-sink/pom.xml
+++ b/ambari-metrics-flume-sink/pom.xml
@@ -169,7 +169,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>14.0.1</version>
+ <version>18.0</version>
</dependency>
</dependencies>
</project>
diff --git a/ambari-metrics-grafana/ambari-metrics/datasource.js b/ambari-metrics-grafana/ambari-metrics/datasource.js
index 9050667..6e14f33 100644
--- a/ambari-metrics-grafana/ambari-metrics/datasource.js
+++ b/ambari-metrics-grafana/ambari-metrics/datasource.js
@@ -484,10 +484,10 @@
}).map(function (uName) {
return uName.value;
});
- selectedUsers = templateSrv._values.Users.lastIndexOf('}') > 0 ? templateSrv._values.Users.slice(1, -1) :
- templateSrv._values.Users;
- var selectedUser = selectedUsers.split(',');
- _.forEach(selectedUser, function (processUser) {
+ if (selectedUsers[0] === "") {
+ selectedUsers = "";
+ }
+ _.forEach(selectedUsers, function (processUser) {
metricsPromises.push(_.map(options.targets, function (target) {
target.hbUser = processUser;
var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
@@ -499,10 +499,18 @@
// Templatized Dashboard for per-table metrics in HBase.
if (templateSrv.variables[0].query === "hbase-tables") {
var splitTables = [];
- var allTables = templateSrv._values.Tables.lastIndexOf('}') > 0 ? templateSrv._values.Tables.slice(1, -1) :
- templateSrv._values.Tables;
- var allTable = allTables.split(',');
- while (allTable.length > 0) {
+ let allTables = [];
+ const tables = templateSrv.index.Tables.options;
+ for (let table of tables) {
+ if (table.text.toLowerCase() === "all" && table.selected) {
+ allTables = "";
+ break;
+ } else if (table.selected) {
+ allTables.push(table.value);
+ }
+ }
+
+ while (allTables.length > 0) {
splitTables.push(allTable.splice(0, 20));
}
_.forEach(splitTables, function (table) {
@@ -530,10 +538,10 @@
}).map(function (topicName) {
return topicName.value;
});
- selectedTopics = templateSrv._values.Topics.lastIndexOf('}') > 0 ? templateSrv._values.Topics.slice(1, -1) :
- templateSrv._values.Topics;
- var selectedTopic = selectedTopics.split(',');
- _.forEach(selectedTopic, function (processTopic) {
+ if (selectedTopics[0] === "") {
+ selectedTopics = "";
+ }
+ _.forEach(selectedTopics, function (processTopic) {
metricsPromises.push(_.map(options.targets, function (target) {
target.kbTopic = processTopic;
target.kbMetric = target.metric.replace('*', target.kbTopic);
@@ -546,15 +554,17 @@
var allCallers = templateSrv.variables.filter(function (variable) {
return variable.query === "callers";
});
- var selectedCallers = (_.isEmpty(allCallers)) ? "" : allCallers[0].options.filter(function (user) {
- return user.selected;
- }).map(function (callerName) {
- return callerName.value;
- });
- selectedCallers = templateSrv._values.Callers.lastIndexOf('}') > 0 ? templateSrv._values.Callers.slice(1, -1) :
- templateSrv._values.Callers;
- var selectedCaller = selectedCallers.split(',');
- _.forEach(selectedCaller, function (processCaller) {
+ let selectedCallers = [];
+ const callers = templateSrv.index.Callers.options;
+ for (let caller of callers) {
+ if (caller.text.toLowerCase() === "all" && caller.selected) {
+ selectedCallers = "";
+ break;
+ } else if (caller.selected) {
+ selectedCallers.push(caller.text);
+ }
+ }
+ _.forEach(selectedCallers, function (processCaller) {
metricsPromises.push(_.map(options.targets, function (target) {
target.nnCaller = processCaller;
target.nnMetric = target.metric.replace('*', target.nnCaller);
@@ -573,10 +583,10 @@
}).map(function (coreName) {
return coreName.value;
});
- selectedCores = templateSrv._values.Cores.lastIndexOf('}') > 0 ? templateSrv._values.Cores.slice(1, -1) :
- templateSrv._values.Cores;
- var selectedCore = selectedCores.split(',');
- _.forEach(selectedCore, function (processCore) {
+ if (selectedCores[0] === "") {
+ selectedCores = "";
+ }
+ _.forEach(selectedCores, function (processCore) {
metricsPromises.push(_.map(options.targets, function (target) {
target.sCore = processCore;
target.sCoreMetric = target.metric.replace('*', target.sCore);
@@ -595,10 +605,10 @@
}).map(function (collectionsName) {
return collectionsName.value;
});
- selectedCollections = templateSrv._values.Collections.lastIndexOf('}') > 0 ? templateSrv._values.Collections.slice(1, -1) :
- templateSrv._values.Collections;
- var selectedCollection = selectedCollections.split(',');
- _.forEach(selectedCollection, function (processCollection) {
+ if (selectedCollections [0] === "") {
+ selectedCollections = "";
+ }
+ _.forEach(selectedCollections, function (processCollection) {
metricsPromises.push(_.map(options.targets, function (target) {
target.sCollection = processCollection;
target.sCollectionMetric = target.metric.replace('*', target.sCollection);
@@ -617,10 +627,10 @@
}).map(function (topoName) {
return topoName.value;
});
- selectedTopologies = templateSrv._values.topologies.lastIndexOf('}') > 0 ? templateSrv._values.topologies.slice(1, -1) :
- templateSrv._values.topologies;
- var selectedTopology = selectedTopologies.split(',');
- _.forEach(selectedTopology, function (processTopology) {
+ if (selectedTopologies === "") {
+ selectedTopologies = "";
+ }
+ _.forEach(selectedTopologies, function (processTopology) {
metricsPromises.push(_.map(options.targets, function (target) {
target.sTopology = processTopology;
target.sTopoMetric = target.metric.replace('*', target.sTopology);
@@ -667,10 +677,10 @@
}).map(function (dataSourceName) {
return dataSourceName.value;
});
- selectedDataSources = templateSrv._values.druidDataSources.lastIndexOf('}') > 0 ? templateSrv._values.druidDataSources.slice(1, -1) :
- templateSrv._values.druidDataSources;
- var selectedDataSource = selectedDataSources.split(',');
- _.forEach(selectedDataSource, function (processDataSource) {
+ if (selectedDataSources[0] === "") {
+ selectedDataSources = "";
+ }
+ _.forEach(selectedDataSources, function (processDataSource) {
metricsPromises.push(_.map(options.targets, function (target) {
target.sDataSource = processDataSource;
target.sDataSourceMetric = target.metric.replace('*', target.sDataSource);
@@ -681,14 +691,21 @@
// To speed up querying on templatized dashboards.
var indexOfHosts = -1;
for (var i = 0; i < templateSrv.variables.length; i++) {
- if (templateSrv.variables[i].name == 'hosts') {
+ if (templateSrv.variables[i].name == 'hosts' && templateSrv.index.hosts) {
indexOfHosts = i;
}
}
if (indexOfHosts >= 0) {
- var allHosts = templateSrv._values.hosts.lastIndexOf('}') > 0 ? templateSrv._values.hosts.slice(1, -1) :
- templateSrv._values.hosts;
- allHosts = templateSrv._texts.hosts === "All" ? '%' : allHosts;
+ let allHosts = [];
+ const hosts = templateSrv.index.hosts.options
+ for (let host of hosts) {
+ if (host.text.toLowerCase() === "all" && host.selected) {
+ allHosts = '%';
+ break;
+ } else if (host.selected) {
+ allHosts.push(host.text);
+ }
+ };
metricsPromises.push(_.map(options.targets, function (target) {
target.templatedHost = allHosts ? allHosts : '';
target.templatedCluster = templatedCluster;
diff --git a/ambari-metrics-host-aggregator/pom.xml b/ambari-metrics-host-aggregator/pom.xml
index 7d8f6b5..38408df 100644
--- a/ambari-metrics-host-aggregator/pom.xml
+++ b/ambari-metrics-host-aggregator/pom.xml
@@ -40,7 +40,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>14.0.1</version>
+ <version>18.0</version>
</dependency>
<dependency>
<groupId>org.apache.ambari</groupId>
diff --git a/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java b/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
index f07d508..cf529ea 100644
--- a/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
+++ b/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
@@ -45,6 +45,7 @@
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
@@ -80,6 +81,7 @@
private static final String TIMELINE_DEFAULT_PROTOCOL = "http";
private static final String EXCLUDED_METRICS_PROPERTY = "external.kafka.metrics.exclude.prefix";
private static final String INCLUDED_METRICS_PROPERTY = "external.kafka.metrics.include.prefix";
+ private static final String INCLUDED_METRICS_REGEX_PROPERTY = "external.kafka.metrics.include.regex";
private volatile boolean initialized = false;
private boolean running = false;
@@ -97,6 +99,7 @@
private String[] excludedMetricsPrefixes;
private String[] includedMetricsPrefixes;
+ private String[] includedMetricsRegex;
// Local cache to avoid prefix matching everytime
private Set<String> excludedMetrics = new HashSet<>();
private boolean hostInMemoryAggregationEnabled;
@@ -214,6 +217,13 @@
includedMetricsPrefixes = includedMetricsStr.trim().split(",");
}
+ // Inclusion override
+ String includedMetricsRegexStr = props.getString(INCLUDED_METRICS_REGEX_PROPERTY, "");
+ if (!StringUtils.isEmpty(includedMetricsRegexStr.trim())) {
+ LOG.info("Including metrics which match the following regex patterns : " + includedMetricsRegexStr);
+ includedMetricsRegex = includedMetricsRegexStr.trim().split(",");
+ }
+
initializeReporter();
if (props.getBoolean(TIMELINE_REPORTER_ENABLED_PROPERTY, false)) {
startReporter(metricsConfig.pollingIntervalSecs());
@@ -273,7 +283,7 @@
", include: " + StringUtils.startsWithAny(metricName, includedMetricsPrefixes));
}
if (StringUtils.startsWithAny(metricName, excludedMetricsPrefixes)) {
- if (!StringUtils.startsWithAny(metricName, includedMetricsPrefixes)) {
+ if (!(StringUtils.startsWithAny(metricName, includedMetricsPrefixes) || Arrays.stream(includedMetricsRegex).anyMatch(metricName::matches))) {
excludedMetrics.add(metricName);
return true;
}
diff --git a/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java b/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
index b05190c..cc56227 100644
--- a/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
+++ b/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
@@ -84,6 +84,7 @@
properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true");
properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c");
properties.setProperty("external.kafka.metrics.include.prefix", "a.b.c.d");
+ properties.setProperty("external.kafka.metrics.include.regex", "a.b.c.*.f");
properties.setProperty("kafka.timeline.metrics.instanceId", "cluster");
properties.setProperty("kafka.timeline.metrics.set.instanceId", "false");
props = new VerifiableProperties(properties);
@@ -118,6 +119,7 @@
properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true");
properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c");
properties.setProperty("external.kafka.metrics.include.prefix", "a.b.c.d");
+ properties.setProperty("external.kafka.metrics.include.regex", "a.b.c.*.f");
properties.setProperty("kafka.timeline.metrics.protocol", "https");
properties.setProperty("kafka.timeline.metrics.truststore.path", "");
properties.setProperty("kafka.timeline.metrics.truststore.type", "");
@@ -143,6 +145,7 @@
Assert.assertFalse(kafkaTimelineMetricsReporter.isExcludedMetric("a.b"));
Assert.assertFalse(kafkaTimelineMetricsReporter.isExcludedMetric("a.b.c.d"));
Assert.assertFalse(kafkaTimelineMetricsReporter.isExcludedMetric("a.b.c.d.e"));
+ Assert.assertFalse(kafkaTimelineMetricsReporter.isExcludedMetric("a.b.c.e.f"));
kafkaTimelineMetricsReporter.stopReporter();
verifyAll();
diff --git a/ambari-metrics-timelineservice/pom.xml b/ambari-metrics-timelineservice/pom.xml
index d185811..cacece9 100644
--- a/ambari-metrics-timelineservice/pom.xml
+++ b/ambari-metrics-timelineservice/pom.xml
@@ -551,7 +551,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>14.0.1</version>
+ <version>18.0</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
@@ -778,7 +778,7 @@
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
- <version>0.11.0.1</version>
+ <version>0.11.0.3</version>
</dependency>
<dependency>
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/RestMetricsSender.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/RestMetricsSender.java
index 455b381..0e707ac 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/RestMetricsSender.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/loadsimulator/net/RestMetricsSender.java
@@ -56,7 +56,7 @@
public String pushMetrics(String payload) {
String responseString = "";
UrlService svc = null;
- Stopwatch timer = new Stopwatch().start();
+ Stopwatch timer = Stopwatch.createStarted();
try {
LOG.info("server: {}", collectorServiceAddress);
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
index c0427e7..b6ff202 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
@@ -121,6 +121,7 @@
import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
import org.apache.ambari.metrics.core.timeline.query.Condition;
import org.apache.ambari.metrics.core.timeline.query.DefaultPhoenixDataSource;
+import org.apache.ambari.metrics.core.timeline.query.MetadataQueryCondition;
import org.apache.ambari.metrics.core.timeline.query.PhoenixConnectionProvider;
import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
import org.apache.ambari.metrics.core.timeline.query.SplitByMetricNamesCondition;
@@ -1990,6 +1991,106 @@
return metadataMap;
}
+ public List<TimelineMetricMetadata> scanMetricMetadataForWildCardRequest(Collection<String> metricNames,
+ String appId,
+ String instanceId) throws SQLException {
+ List<TimelineMetricMetadata> metadataList = new ArrayList<>();
+ Connection conn = getConnection();
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+
+ MetadataQueryCondition metadataQueryCondition = new MetadataQueryCondition(new ArrayList<>(metricNames), appId, instanceId);
+ stmt = PhoenixTransactSQL.prepareScanMetricMetadataSqlStmt(conn, metadataQueryCondition);
+ try {
+ if (stmt != null) {
+ rs = stmt.executeQuery();
+ while (rs.next()) {
+ TimelineMetricMetadata metadata = new TimelineMetricMetadata(
+ rs.getString("METRIC_NAME"),
+ rs.getString("APP_ID"),
+ rs.getString("INSTANCE_ID"),
+ null,
+ null,
+ null,
+ false,
+ true
+ );
+
+ metadata.setUuid(rs.getBytes("UUID"));
+ metadataList.add(metadata);
+ }
+ }
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+
+ return metadataList;
+ }
+
+ public List<byte[]> scanHostMetadataForWildCardRequest(List<String> hostnames) throws SQLException {
+ List<byte[]> uuidList = new ArrayList<>();
+ Connection conn = getConnection();
+ PreparedStatement stmt;
+ ResultSet rs = null;
+
+ MetadataQueryCondition metadataQueryCondition = new MetadataQueryCondition(hostnames);
+ stmt = PhoenixTransactSQL.prepareScanMetricMetadataSqlStmt(conn, metadataQueryCondition);
+ try {
+ if (stmt != null) {
+ rs = stmt.executeQuery();
+ while (rs.next()) {
+ byte[] uuid = rs.getBytes("UUID");
+ if (uuid != null) {
+ uuidList.add(uuid);
+ }
+ }
+ }
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ // Ignore
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException sql) {
+ // Ignore
+ }
+ }
+ }
+ return uuidList;
+ }
+
// No filter criteria support for now.
public Map<TimelineMetricMetadataKey, TimelineMetricMetadata> getTimelineMetricMetadataV1() throws SQLException {
Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataMap = new HashMap<>();
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
index 2dedd77..7c6fce5 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
@@ -39,11 +39,11 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import org.apache.ambari.metrics.core.timeline.MetricsSystemInitializationException;
import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
import org.apache.ambari.metrics.core.timeline.uuid.MetricUuidGenStrategy;
-import org.apache.ambari.metrics.core.timeline.uuid.MD5UuidGenStrategy;
import org.apache.ambari.metrics.core.timeline.uuid.Murmur3HashUuidGenStrategy;
import org.apache.ambari.metrics.core.timeline.uuid.TimelineMetricUuid;
import org.apache.commons.collections.CollectionUtils;
@@ -64,7 +64,6 @@
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TRANSIENT_METRIC_PATTERNS;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.METRICS_METADATA_SYNC_INIT_DELAY;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.METRICS_METADATA_SYNC_SCHEDULE_DELAY;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_UUID_GEN_STRATEGY;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_METADATA_FILTERS;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_HOSTED_APPS_METADATA_TABLE_SQL;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_INSTANCE_HOST_TABLE_SQL;
@@ -361,7 +360,8 @@
}
boolean isDistributedModeEnabled() {
- return metricsConf.get("timeline.metrics.service.operation.mode").equals("distributed");
+ String mode = metricsConf.get("timeline.metrics.service.operation.mode");
+ return (mode != null) && mode.equals("distributed");
}
/**
@@ -629,75 +629,99 @@
String instanceId,
List<String> transientMetricNames) {
- Collection<String> sanitizedMetricNames = new HashSet<>();
List<byte[]> uuids = new ArrayList<>();
+ boolean metricNameHasWildcard = false;
for (String metricName : metricNames) {
- if (metricName.contains("%")) {
- String metricRegEx = getJavaRegexFromSqlRegex(metricName);
- for (TimelineMetricMetadataKey key : METADATA_CACHE.keySet()) {
- String metricNameFromMetadata = key.getMetricName();
- if (metricNameFromMetadata.matches(metricRegEx)) {
- sanitizedMetricNames.add(metricNameFromMetadata);
- }
- }
- } else {
- sanitizedMetricNames.add(metricName);
+ if (hasWildCard(metricName)) {
+ metricNameHasWildcard = true;
+ break;
}
}
- if(sanitizedMetricNames.isEmpty()) {
- return uuids;
+ boolean hostNameHasWildcard = false;
+ if (CollectionUtils.isNotEmpty(hostnames)) {
+ for (String hostname : hostnames) {
+ if (hasWildCard(hostname)) {
+ hostNameHasWildcard = true;
+ break;
+ }
+ }
}
- Set<String> sanitizedHostNames = getSanitizedHostnames(hostnames);
-
if ( StringUtils.isNotEmpty(appId) && !(appId.equals("HOST") || appId.equals("FLUME_HANDLER"))) { //HACK.. Why??
appId = appId.toLowerCase();
}
- if (CollectionUtils.isNotEmpty(sanitizedHostNames)) {
- if (CollectionUtils.isNotEmpty(sanitizedMetricNames)) {
- //Skip getting UUID if it is a transient metric.
- //An attempt to get it will also be OK as we don't add null UUIDs.
- for (String metricName : sanitizedMetricNames) {
- if (isTransientMetric(metricName, appId)) {
- transientMetricNames.add(metricName);
- continue;
+ if (hasWildCard(instanceId) || hasWildCard(appId) || hostNameHasWildcard || metricNameHasWildcard) {
+ try {
+ List<TimelineMetricMetadata> metricMetadataFromStore = hBaseAccessor.scanMetricMetadataForWildCardRequest(metricNames,
+ appId, instanceId);
+ List<byte[]> hostUuidsFromStore = hBaseAccessor.scanHostMetadataForWildCardRequest(hostnames);
+
+ for (TimelineMetricMetadata matchedEntry : metricMetadataFromStore) {
+ if (matchedEntry.getUuid() != null) {
+ if (CollectionUtils.isNotEmpty(hostnames)) {
+ for (byte[] hostUuidEntry : hostUuidsFromStore) {
+ uuids.add(ArrayUtils.addAll(matchedEntry.getUuid(), hostUuidEntry));
+ }
+ } else {
+ uuids.add(matchedEntry.getUuid());
+ }
+ } else if (isTransientMetric(matchedEntry.getMetricName(), matchedEntry.getAppId())) {
+ transientMetricNames.add(matchedEntry.getMetricName());
}
- TimelineMetric metric = new TimelineMetric();
- metric.setMetricName(metricName);
- metric.setAppId(appId);
- metric.setInstanceId(instanceId);
- for (String hostname : sanitizedHostNames) {
- metric.setHostName(hostname);
- byte[] uuid = getUuid(metric, false);
+ }
+ return uuids;
+ } catch (SQLException e) {
+ LOG.error("Unable to query metadata table to check satisfying metric keys for wildcard request : " + e);
+ return uuids;
+ }
+ } else {
+
+ if (CollectionUtils.isNotEmpty(hostnames)) {
+ if (CollectionUtils.isNotEmpty(metricNames)) {
+ //Skip getting UUID if it is a transient metric.
+ //An attempt to get it will also be OK as we don't add null UUIDs.
+ for (String metricName : metricNames) {
+ if (isTransientMetric(metricName, appId)) {
+ transientMetricNames.add(metricName);
+ continue;
+ }
+ TimelineMetric metric = new TimelineMetric();
+ metric.setMetricName(metricName);
+ metric.setAppId(appId);
+ metric.setInstanceId(instanceId);
+ for (String hostname : hostnames) {
+ metric.setHostName(hostname);
+ byte[] uuid = getUuid(metric, false);
+ if (uuid != null) {
+ uuids.add(uuid);
+ }
+ }
+ }
+ } else {
+ for (String hostname : hostnames) {
+ byte[] uuid = getUuidForHostname(hostname, false);
if (uuid != null) {
uuids.add(uuid);
}
}
}
} else {
- for (String hostname : sanitizedHostNames) {
- byte[] uuid = getUuidForHostname(hostname, false);
+ for (String metricName : metricNames) {
+ //Skip getting UUID if it is a transient metric. An attempt to get it will also be OK as we don't add null UUIDs.
+ if (isTransientMetric(metricName, appId)) {
+ transientMetricNames.add(metricName);
+ continue;
+ }
+ TimelineClusterMetric metric = new TimelineClusterMetric(metricName, appId, instanceId, -1l);
+ byte[] uuid = getUuid(metric, false);
if (uuid != null) {
uuids.add(uuid);
}
}
}
- } else {
- for (String metricName : sanitizedMetricNames) {
- //Skip getting UUID if it is a transient metric. An attempt to get it will also be OK as we don't add null UUIDs.
- if (isTransientMetric(metricName, appId)) {
- transientMetricNames.add(metricName);
- continue;
- }
- TimelineClusterMetric metric = new TimelineClusterMetric(metricName, appId, instanceId, -1l);
- byte[] uuid = getUuid(metric, false);
- if (uuid != null) {
- uuids.add(uuid);
- }
- }
}
return uuids;
@@ -833,6 +857,9 @@
* Run TimelineMetricMetadataSync once
*/
public void forceMetricsMetadataSync() {
+ if (metricMetadataSync == null) {
+ metricMetadataSync = new TimelineMetricMetadataSync(this);
+ }
metricMetadataSync.run();
}
@@ -860,4 +887,9 @@
}
}
}
+
+ private boolean hasWildCard(String key) {
+ return StringUtils.isNotEmpty(key) && key.contains("%");
+ }
+
}
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultCondition.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultCondition.java
index 7f72187..46204fc 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultCondition.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/DefaultCondition.java
@@ -28,7 +28,6 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.metrics2.sink.timeline.Precision;
import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
-import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
public class DefaultCondition implements Condition {
List<String> metricNames;
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/MetadataQueryCondition.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/MetadataQueryCondition.java
new file mode 100644
index 0000000..f6e8c6a
--- /dev/null
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/MetadataQueryCondition.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.core.timeline.query;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+
+public class MetadataQueryCondition extends TransientMetricCondition {
+
+ private boolean isMetricMetadataCondition = true;
+
+ public MetadataQueryCondition(List<String> metricNames, String appId, String instanceId) {
+ super(metricNames, Collections.EMPTY_LIST, appId, instanceId, null, null, null , null, true, null);
+ this.hostnames = Collections.EMPTY_LIST;
+ }
+
+ public MetadataQueryCondition(List<String> hostnames) {
+ super(Collections.EMPTY_LIST, hostnames, StringUtils.EMPTY, StringUtils.EMPTY, null, null, null , null, true, null);
+ isMetricMetadataCondition = false;
+ }
+
+ public StringBuilder getConditionClause() {
+ StringBuilder sb = new StringBuilder();
+
+ boolean appendConjunction = false;
+ if (CollectionUtils.isNotEmpty(metricNames)) {
+ appendConjunction = appendMetricNameClause(sb);
+ }
+
+ if (CollectionUtils.isNotEmpty(hostnames)) {
+ appendConjunction = appendHostnameClause(sb, appendConjunction);
+ }
+
+ String appId = getAppId();
+ if (StringUtils.isNotEmpty(appId)) {
+ if (appId.contains("%")) {
+ appendConjunction = append(sb, appendConjunction, appId, " APP_ID LIKE ?");
+ } else {
+ appendConjunction = append(sb, appendConjunction, appId, " APP_ID = ?");
+ }
+ }
+
+ String instanceId = getInstanceId();
+ if (StringUtils.isNotEmpty(instanceId)) {
+ if (instanceId.contains("%")) {
+ appendConjunction = append(sb, appendConjunction, instanceId, " INSTANCE_ID LIKE ?");
+ } else {
+ appendConjunction = append(sb, appendConjunction, instanceId, " INSTANCE_ID = ?");
+ }
+ }
+
+ return sb;
+ }
+
+ public boolean isMetricMetadataCondition() {
+ return isMetricMetadataCondition;
+ }
+}
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
index 6a1d03f..ee50051 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
@@ -377,6 +377,10 @@
public static final String GET_INSTANCE_HOST_METADATA_SQL = "SELECT " +
"INSTANCE_ID, HOSTNAME FROM INSTANCE_HOST_METADATA";
+ public static final String SCAN_METRIC_METADATA_SQL = "SELECT METRIC_NAME, APP_ID, INSTANCE_ID, UUID FROM %s";
+
+ public static final String SCAN_HOST_METADATA_SQL = "SELECT HOSTNAME, UUID FROM %s";
+
/**
* Aggregate host metrics using a GROUP BY clause to take advantage of
* N - way parallel scan where N = number of regions.
@@ -462,6 +466,12 @@
public static final String METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME =
"METRIC_AGGREGATE_DAILY_UUID";
+ public static final String METRICS_METADATA_TABLE_NAME =
+ "METRICS_METADATA_UUID";
+
+ public static final String HOST_METADATA_TABLE_NAME =
+ "HOSTED_APPS_METADATA_UUID";
+
public static final Pattern PHOENIX_TABLES_REGEX_PATTERN = Pattern.compile("METRIC_.*");
public static final String[] PHOENIX_TABLES = {
@@ -980,6 +990,46 @@
}
+ public static PreparedStatement prepareScanMetricMetadataSqlStmt(Connection connection, MetadataQueryCondition condition)
+ throws SQLException {
+
+ String stmtStr;
+ if (condition.isMetricMetadataCondition()) {
+ stmtStr = String.format(SCAN_METRIC_METADATA_SQL, METRICS_METADATA_TABLE_NAME);
+ } else {
+ stmtStr = String.format(SCAN_HOST_METADATA_SQL, HOST_METADATA_TABLE_NAME);
+ }
+
+ StringBuilder sb = new StringBuilder(stmtStr);
+
+ sb.append(" WHERE ");
+ sb.append(condition.getConditionClause());
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL: " + sb.toString() + ", condition: " + condition);
+ }
+
+ PreparedStatement stmt = null;
+ try {
+ stmt = connection.prepareStatement(sb.toString());
+ int pos = 1;
+ if (condition.isMetricMetadataCondition()) {
+ pos = addMetricNames(condition, pos, stmt);
+ pos = addAppId(condition, pos, stmt);
+ pos = addInstanceId(condition, pos, stmt);
+ } else {
+ pos = addHostNames(condition, pos, stmt);
+ }
+
+ } catch (SQLException e) {
+ if (stmt != null) {
+ stmt.close();
+ }
+ throw e;
+ }
+ return stmt;
+ }
+
public static String getTargetTableUsingPrecision(Precision precision, boolean withHosts) {
String inputTable = null;
diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java
index c26d18d..b0f526c 100644
--- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java
+++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java
@@ -42,6 +42,16 @@
}
}
+ public TransientMetricCondition(List<String> metricNames, List<String> hostnames, String appId,
+ String instanceId, Long startTime, Long endTime, Precision precision,
+ Integer limit, boolean grouped, List<String> transientMetricNames) {
+ super(metricNames, hostnames, appId, instanceId, startTime, endTime, precision, limit, grouped);
+ this.transientMetricNames = transientMetricNames;
+ if (CollectionUtils.isEmpty(hostnames)) {
+ this.hostnames = Collections.singletonList("%");
+ }
+ }
+
public StringBuilder getTransientConditionClause() {
StringBuilder sb = new StringBuilder();
@@ -49,15 +59,26 @@
appendConjunction = appendHostnameClause(sb, appendConjunction);
- appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID = ?");
- appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID = ?");
+ String appId = getAppId();
+ if (appId.contains("%")) {
+ appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID LIKE ?");
+ } else {
+ appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID = ?");
+ }
+
+ String instanceId = getInstanceId();
+ if (instanceId.contains("%")) {
+ appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID LIKE ?");
+ } else {
+ appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID = ?");
+ }
appendConjunction = append(sb, appendConjunction, getStartTime(), " SERVER_TIME >= ?");
append(sb, appendConjunction, getEndTime(), " SERVER_TIME < ?");
return sb;
}
- private boolean appendMetricNameClause(StringBuilder sb) {
+ protected boolean appendMetricNameClause(StringBuilder sb) {
boolean appendConjunction = false;
List<String> metricsLike = new ArrayList<>();
List<String> metricsIn = new ArrayList<>();
diff --git a/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java b/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java
index 90def86..90a5d10 100644
--- a/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java
+++ b/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java
@@ -181,7 +181,7 @@
}
@Test
- public void testWildcardSanitization() throws IOException, SQLException, URISyntaxException {
+ public void testMetricHostWildcardSanitization() throws IOException, SQLException, URISyntaxException {
// Initialize new manager
metadataManager = new TimelineMetricMetadataManager(new Configuration(), hdb);
hdb.setMetadataInstance(metadataManager);
@@ -234,6 +234,7 @@
replay(configuration);
hdb.insertMetricRecordsWithMetadata(metadataManager, timelineMetrics, true);
+ metadataManager.forceMetricsMetadataSync();
List<byte[]> uuids = metadataManager.getUuidsForGetMetricQuery(Collections.singletonList("dummy_m%"),
Collections.singletonList("dummy_host2"), "dummy_app1", null);
@@ -255,6 +256,85 @@
}
@Test
+ public void testAppInstanceWildcardSanitization() throws IOException, SQLException, URISyntaxException {
+ // Initialize new manager
+ metadataManager = new TimelineMetricMetadataManager(new Configuration(), hdb);
+ hdb.setMetadataInstance(metadataManager);
+ final long now = System.currentTimeMillis();
+
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+
+ TimelineMetric metric1 = new TimelineMetric();
+ metric1.setMetricName("dummy_m1");
+ metric1.setHostName("dummy_host1");
+ metric1.setStartTime(now - 1000);
+ metric1.setAppId("dummy_app1");
+ metric1.setInstanceId("dummy_i1");
+ metric1.setType("Integer");
+ metric1.setMetricValues(new TreeMap<Long, Double>() {{
+ put(now - 100, 1.0);
+ put(now - 200, 2.0);
+ put(now - 300, 3.0);
+ }});
+ timelineMetrics.getMetrics().add(metric1);
+
+ TimelineMetric metric2 = new TimelineMetric();
+ metric2.setMetricName("dummy_m2");
+ metric2.setHostName("dummy_host2");
+ metric2.setStartTime(now - 1000);
+ metric2.setAppId("dummy_app2");
+ metric2.setInstanceId("dummy_i2");
+ metric2.setType("Integer");
+ metric2.setMetricValues(new TreeMap<Long, Double>() {{
+ put(now - 100, 1.0);
+ put(now - 200, 2.0);
+ put(now - 300, 3.0);
+ }});
+ timelineMetrics.getMetrics().add(metric2);
+
+ TimelineMetric metric3 = new TimelineMetric();
+ metric3.setMetricName("dummy_3");
+ metric3.setHostName("gummy_3h");
+ metric3.setStartTime(now - 1000);
+ metric3.setAppId("gummy_app3");
+ metric3.setType("Integer");
+ metric3.setMetricValues(new TreeMap<Long, Double>() {{
+ put(now - 100, 1.0);
+ put(now - 200, 2.0);
+ put(now - 300, 3.0);
+ }});
+ timelineMetrics.getMetrics().add(metric3);
+
+ Configuration metricsConf = new Configuration();
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+ replay(configuration);
+
+ hdb.insertMetricRecordsWithMetadata(metadataManager, timelineMetrics, true);
+ metadataManager.forceMetricsMetadataSync();
+
+ List<byte[]> uuids = metadataManager.getUuidsForGetMetricQuery(Collections.singletonList("dummy_m%"),
+ Collections.singletonList("dummy_host%"), "dummy_app%", "dummy_i%");
+ Assert.assertTrue(uuids.size() == 4);
+
+ uuids = metadataManager.getUuidsForGetMetricQuery(Collections.singletonList("dummy_m1"),
+ Collections.singletonList("dummy_host1"), "dummy_app1", "%");
+ Assert.assertTrue(uuids.size() == 1);
+
+ uuids = metadataManager.getUuidsForGetMetricQuery(Collections.singletonList("dummy_m1"),
+ Collections.singletonList("dummy_host1"), "dummy_app1", "dummy_i%");
+ Assert.assertTrue(uuids.size() == 1);
+
+ uuids = metadataManager.getUuidsForGetMetricQuery(Collections.singletonList("dummy_m1"),
+ Collections.singletonList("dummy_host1"), "dummy_app1", "dummy_i2");
+ Assert.assertTrue(uuids.size() == 0);
+
+ uuids = metadataManager.getUuidsForGetMetricQuery(Collections.singletonList("%"),
+ Collections.singletonList("%"), "%", "%");
+ Assert.assertTrue(uuids.size() == 6);
+ }
+
+ @Test
public void testTransientMetricPatterns() {
long now = System.currentTimeMillis();