HDDS-7045. Election info is out of date in Recon (#3677)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
index 90ac003..e5122f8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
@@ -52,7 +52,7 @@
"0.0.0.0";
public static final int OZONE_RECON_DATANODE_PORT_DEFAULT = 9891;
// Prometheus HTTP endpoint including port
- // ex: http://localhost:9090
+ // ex: http://prometheus:9090
public static final String OZONE_RECON_PROMETHEUS_HTTP_ENDPOINT =
"ozone.recon.prometheus.http.endpoint";
/**
diff --git a/hadoop-hdds/docs/content/concept/Recon.md b/hadoop-hdds/docs/content/concept/Recon.md
index 064127a..b5d4731 100644
--- a/hadoop-hdds/docs/content/concept/Recon.md
+++ b/hadoop-hdds/docs/content/concept/Recon.md
@@ -109,7 +109,7 @@
and can display useful information in Recon UI in Datanodes and Pipelines pages.
Recon also exposes a proxy endpoint ([/metrics]({{< ref "interface/ReconApi.md#metrics" >}}))
to query Prometheus. This integration can be enabled by setting this configuration `ozone.recon.prometheus.http.endpoint`
-to the Prometheus endpoint like `ozone.recon.prometheus.http.endpoint=localhost:9090`.
+to the Prometheus endpoint like `ozone.recon.prometheus.http.endpoint=http://prometheus:9090`.
## API Reference
diff --git a/hadoop-hdds/docs/content/concept/Recon.zh.md b/hadoop-hdds/docs/content/concept/Recon.zh.md
index 5c67351..3880cfd 100644
--- a/hadoop-hdds/docs/content/concept/Recon.zh.md
+++ b/hadoop-hdds/docs/content/concept/Recon.zh.md
@@ -70,7 +70,7 @@
## Recon 和 Prometheus
-Recon 可以与任何配置为收集指标的 Prometheus 实例集成，并且可以在数据节点和 Pipelines 页面的 Recon UI 中显示有用的信息。Recon 还公开了一个代理端点 ([/指标]({{< ref path="interface/ReconApi.zh.md#metrics" >}})) 来查询 Prometheus。可以通过将此配置`ozone.recon.prometheus.http.endpoint`设置为 Prometheus 端点如`ozone.recon.prometheus.http.endpoint=localhost:9090`来启用此集成。
+Recon 可以与任何配置为收集指标的 Prometheus 实例集成，并且可以在数据节点和 Pipelines 页面的 Recon UI 中显示有用的信息。Recon 还公开了一个代理端点 ([/指标]({{< ref path="interface/ReconApi.zh.md#metrics" >}})) 来查询 Prometheus。可以通过将此配置`ozone.recon.prometheus.http.endpoint`设置为 Prometheus 端点如`ozone.recon.prometheus.http.endpoint=http://prometheus:9090`来启用此集成。
## API 参考
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java
index 0595bbf..c5cc6d9 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java
@@ -39,7 +39,6 @@
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
-import java.util.Optional;
import java.util.TreeMap;
import java.util.UUID;
@@ -114,12 +113,12 @@
// is group-79DF51EE872D
String[] splits = pipelineId.toString().split("-");
String groupId = "group-" + splits[splits.length - 1].toUpperCase();
- Optional<Long> leaderElectionCount = getMetricValue(
- "ratis_leader_election_electionCount", groupId);
- leaderElectionCount.ifPresent(pipelineBuilder::setLeaderElections);
- Optional<Long> leaderElectionTime = getMetricValue(
- "ratis_leader_election_lastLeaderElectionTime", groupId);
- leaderElectionTime.ifPresent(pipelineBuilder::setLastLeaderElection);
+ Long leaderElectionCount = getElectionCountMetricValue(groupId);
+ pipelineBuilder.setLeaderElections(leaderElectionCount);
+ Long lastLeaderElectionElapsedTime
+ = getLastLeaderElectionElapsedTimeMetricValue(groupId,
+ pipeline.getLeaderId());
+ pipelineBuilder.setLastLeaderElection(lastLeaderElectionElapsedTime);
}
pipelinesList.add(pipelineBuilder.build());
@@ -130,25 +129,53 @@
return Response.ok(pipelinesResponse).build();
}
- private Optional<Long> getMetricValue(String metricName, String groupId) {
+ private Long getElectionCountMetricValue(String groupId) {
+ Long electionCount = 0L;
String metricsQuery = String.format(
- "query=%s{group=\"%s\"}", metricName, groupId);
+ "query=ratis_leader_election_electionCount{group=\"%s\"}", groupId);
try {
List<Metric> metrics = metricsServiceProvider.getMetricsInstant(
- metricsQuery);
+ metricsQuery);
if (!metrics.isEmpty()) {
- TreeMap<Double, Double> values = (TreeMap<Double, Double>)
- metrics.get(0).getValues();
- if (!values.isEmpty()) {
- return Optional.of(values.firstEntry().getValue().longValue());
+ for (Metric m : metrics) {
+ TreeMap<Double, Double> values = (TreeMap<Double, Double>)
+ m.getValues();
+ if (!values.isEmpty()) {
+ electionCount += values.firstEntry().getValue().longValue();
+ }
}
}
} catch (Exception ex) {
if (LOG.isErrorEnabled()) {
- LOG.error(String.format("Unable to get metrics value for %s",
- metricName), ex);
+ LOG.error("Unable to get metrics value for " +
+ "ratis_leader_election_electionCount", ex);
}
}
- return Optional.empty();
+ return electionCount;
+ }
+
+ private Long getLastLeaderElectionElapsedTimeMetricValue(String groupId,
+ UUID uuid) {
+ String metricsQuery = String.format(
+ "query=ratis_leader_election_lastLeaderElectionElapsedTime{group=" +
+ "\"%s\",exported_instance=\"%s\"}", groupId,
+ uuid.toString());
+ try {
+ List<Metric> metrics = metricsServiceProvider.getMetricsInstant(
+ metricsQuery);
+ if (!metrics.isEmpty()) {
+ TreeMap<Double, Double> values = (TreeMap<Double, Double>)
+ metrics.get(0).getValues();
+ if (!values.isEmpty()) {
+ return values.firstEntry().getValue().longValue();
+ }
+ }
+ } catch (Exception ex) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Unable to get metrics value for " +
+ "ratis_leader_election_lastLeaderElectionElapsedTime", ex);
+ }
+ }
+ return 0L;
}
}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx
index 6810cba..a71559d 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx
@@ -18,7 +18,7 @@
import React from 'react';
import axios from 'axios';
-import {Table, Tabs} from 'antd';
+import {Table, Tabs, Tooltip, Icon} from 'antd';
import './pipelines.less';
import {PaginationConfig} from 'antd/lib/pagination';
import prettyMilliseconds from 'pretty-ms';
@@ -120,11 +120,17 @@
sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.leaderNode.localeCompare(b.leaderNode)
},
{
- title: 'Last Leader Election',
+ title:
+ <span>
+ Last Leader Election
+ <Tooltip title='Elapsed time since the current leader got elected. Only available if any metrics service providers like Prometheus is configured.'>
+ <Icon type='info-circle'/>
+ </Tooltip>
+ </span>,
dataIndex: 'lastLeaderElection',
key: 'lastLeaderElection',
render: (lastLeaderElection: number) => lastLeaderElection > 0 ?
- moment(lastLeaderElection).format('ll LTS') : 'NA',
+ prettyMilliseconds(lastLeaderElection, {compact: true}) + " ago" : 'NA',
sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.lastLeaderElection - b.lastLeaderElection
},
{
@@ -135,10 +141,18 @@
sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.duration - b.duration
},
{
- title: 'No. of Elections',
+ title:
+ <span>
+ No. of Elections
+ <Tooltip title='Number of elections in this pipeline. Only available if any metrics service providers like Prometheus is configured.'>
+ <Icon type='info-circle'/>
+ </Tooltip>
+ </span>,
dataIndex: 'leaderElections',
key: 'leaderElections',
isSearchable: true,
+ render: (leaderElections: number) => leaderElections > 0 ?
+ leaderElections : 'NA',
sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.leaderElections - b.leaderElections
}
];
