HDFS-12273. Federation UI. Contributed by Inigo Goiri.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 0fe491be..65eea31 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -259,6 +259,9 @@
<copy file="${basedir}/src/main/webapps/proto-web.xml"
tofile="${project.build.directory}/webapps/nfs3/WEB-INF/web.xml"
filtering="true"/>
+ <copy file="${basedir}/src/main/webapps/proto-web.xml"
+ tofile="${project.build.directory}/webapps/router/WEB-INF/web.xml"
+ filtering="true"/>
<copy toDir="${project.build.directory}/webapps">
<fileset dir="${basedir}/src/main/webapps">
<exclude name="**/proto-web.xml"/>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 48521a3..1f96763 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1252,6 +1252,25 @@
FEDERATION_ROUTER_PREFIX + "admin.enable";
public static final boolean DFS_ROUTER_ADMIN_ENABLE_DEFAULT = true;
+ // HDFS Router-based federation web
+ public static final String DFS_ROUTER_HTTP_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "http.enable";
+ public static final boolean DFS_ROUTER_HTTP_ENABLE_DEFAULT = true;
+ public static final String DFS_ROUTER_HTTP_ADDRESS_KEY =
+ FEDERATION_ROUTER_PREFIX + "http-address";
+ public static final int DFS_ROUTER_HTTP_PORT_DEFAULT = 50071;
+ public static final String DFS_ROUTER_HTTP_BIND_HOST_KEY =
+ FEDERATION_ROUTER_PREFIX + "http-bind-host";
+ public static final String DFS_ROUTER_HTTP_ADDRESS_DEFAULT =
+ "0.0.0.0:" + DFS_ROUTER_HTTP_PORT_DEFAULT;
+ public static final String DFS_ROUTER_HTTPS_ADDRESS_KEY =
+ FEDERATION_ROUTER_PREFIX + "https-address";
+ public static final int DFS_ROUTER_HTTPS_PORT_DEFAULT = 50072;
+ public static final String DFS_ROUTER_HTTPS_BIND_HOST_KEY =
+ FEDERATION_ROUTER_PREFIX + "https-bind-host";
+ public static final String DFS_ROUTER_HTTPS_ADDRESS_DEFAULT =
+ "0.0.0.0:" + DFS_ROUTER_HTTPS_PORT_DEFAULT;
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
index 43efb3c..cb4245a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
@@ -201,4 +201,11 @@
* @return Host and port of the router.
*/
String getBlockPoolId();
+
+ /**
+ * Get the current state of the router.
+ *
+ * @return String label for the current router state.
+ */
+ String getRouterStatus();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
index 1e80256..7844a2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
@@ -21,6 +21,9 @@
import java.io.IOException;
import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
@@ -34,6 +37,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.function.ToIntFunction;
import java.util.function.ToLongFunction;
@@ -83,6 +87,9 @@
/** Format for a date. */
private static final String DATE_FORMAT = "yyyy/MM/dd HH:mm:ss";
+ /** Prevent holding the page from load too long. */
+ private static final long TIME_OUT = TimeUnit.SECONDS.toMillis(1);
+
/** Router interface. */
private final Router router;
@@ -353,8 +360,8 @@
final Map<String, Map<String, Object>> info = new HashMap<>();
try {
RouterRpcServer rpcServer = this.router.getRpcServer();
- DatanodeInfo[] live =
- rpcServer.getDatanodeReport(DatanodeReportType.LIVE);
+ DatanodeInfo[] live = rpcServer.getDatanodeReport(
+ DatanodeReportType.LIVE, TIME_OUT);
if (live.length > 0) {
float totalDfsUsed = 0;
@@ -446,7 +453,14 @@
@Override
public String getHostAndPort() {
- // TODO this should be the HTTP address
+ InetSocketAddress address = this.router.getHttpServerAddress();
+ if (address != null) {
+ try {
+ String hostname = InetAddress.getLocalHost().getHostName();
+ int port = address.getPort();
+ return hostname + ":" + port;
+ } catch (UnknownHostException ignored) { }
+ }
return "Unknown";
}
@@ -479,6 +493,11 @@
}
}
+ @Override
+ public String getRouterStatus() {
+ return "RUNNING";
+ }
+
/**
* Build a set of unique values found in all namespaces.
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
index d974c78..0950cde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
@@ -43,6 +43,7 @@
import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -222,6 +223,28 @@
report.getServiceAddress(), report.getLifelineAddress(),
report.getWebAddress(), report.getState(), report.getSafemode());
+ if (report.statsValid()) {
+ MembershipStats stats = MembershipStats.newInstance();
+ stats.setNumOfFiles(report.getNumFiles());
+ stats.setNumOfBlocks(report.getNumBlocks());
+ stats.setNumOfBlocksMissing(report.getNumBlocksMissing());
+ stats.setNumOfBlocksPendingReplication(
+ report.getNumOfBlocksPendingReplication());
+ stats.setNumOfBlocksUnderReplicated(
+ report.getNumOfBlocksUnderReplicated());
+ stats.setNumOfBlocksPendingDeletion(
+ report.getNumOfBlocksPendingDeletion());
+ stats.setAvailableSpace(report.getAvailableSpace());
+ stats.setTotalSpace(report.getTotalSpace());
+ stats.setNumOfDecommissioningDatanodes(
+ report.getNumDecommissioningDatanodes());
+ stats.setNumOfActiveDatanodes(report.getNumLiveDatanodes());
+ stats.setNumOfDeadDatanodes(report.getNumDeadDatanodes());
+ stats.setNumOfDecomActiveDatanodes(report.getNumDecomLiveDatanodes());
+ stats.setNumOfDecomDeadDatanodes(report.getNumDecomDeadDatanodes());
+ record.setStats(stats);
+ }
+
if (report.getState() != UNAVAILABLE) {
// Set/update our last contact time
record.setLastContact(Time.now());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index 3ab5e2d..df5549c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -89,6 +89,9 @@
private RouterAdminServer adminServer;
private InetSocketAddress adminAddress;
+ /** HTTP interface and web application. */
+ private RouterHttpServer httpServer;
+
/** Interface with the State Store. */
private StateStoreService stateStore;
@@ -169,6 +172,14 @@
}
if (conf.getBoolean(
+ DFSConfigKeys.DFS_ROUTER_HTTP_ENABLE,
+ DFSConfigKeys.DFS_ROUTER_HTTP_ENABLE_DEFAULT)) {
+ // Create HTTP server
+ this.httpServer = createHttpServer();
+ addService(this.httpServer);
+ }
+
+ if (conf.getBoolean(
DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE_DEFAULT)) {
@@ -354,6 +365,31 @@
}
/////////////////////////////////////////////////////////
+ // HTTP server
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Create an HTTP server for this Router.
+ *
+ * @return HTTP server for this Router.
+ */
+ protected RouterHttpServer createHttpServer() {
+ return new RouterHttpServer(this);
+ }
+
+ /**
+ * Get the current HTTP socket address for the router.
+ *
+ * @return InetSocketAddress HTTP address.
+ */
+ public InetSocketAddress getHttpServerAddress() {
+ if (httpServer != null) {
+ return httpServer.getHttpAddress();
+ }
+ return null;
+ }
+
+ /////////////////////////////////////////////////////////
// Namenode heartbeat monitors
/////////////////////////////////////////////////////////
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
new file mode 100644
index 0000000..046f0ba
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.service.AbstractService;
+
+/**
+ * Web interface for the {@link Router}. It exposes the Web UI and the WebHDFS
+ * methods from {@link RouterWebHdfsMethods}.
+ */
+public class RouterHttpServer extends AbstractService {
+
+ protected static final String NAMENODE_ATTRIBUTE_KEY = "name.node";
+
+
+ /** Configuration for the Router HTTP server. */
+ private Configuration conf;
+
+ /** Router using this HTTP server. */
+ private final Router router;
+
+ /** HTTP server. */
+ private HttpServer2 httpServer;
+
+ /** HTTP addresses. */
+ private InetSocketAddress httpAddress;
+ private InetSocketAddress httpsAddress;
+
+
+ public RouterHttpServer(Router router) {
+ super(RouterHttpServer.class.getName());
+ this.router = router;
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ this.conf = configuration;
+
+ // Get HTTP address
+ this.httpAddress = conf.getSocketAddr(
+ DFSConfigKeys.DFS_ROUTER_HTTP_BIND_HOST_KEY,
+ DFSConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY,
+ DFSConfigKeys.DFS_ROUTER_HTTP_ADDRESS_DEFAULT,
+ DFSConfigKeys.DFS_ROUTER_HTTP_PORT_DEFAULT);
+
+ // Get HTTPs address
+ this.httpsAddress = conf.getSocketAddr(
+ DFSConfigKeys.DFS_ROUTER_HTTPS_BIND_HOST_KEY,
+ DFSConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY,
+ DFSConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_DEFAULT,
+ DFSConfigKeys.DFS_ROUTER_HTTPS_PORT_DEFAULT);
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ // Build and start server
+ String webApp = "router";
+ HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(
+ this.conf, this.httpAddress, this.httpsAddress, webApp,
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY);
+
+ this.httpServer = builder.build();
+
+ this.httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, this.router);
+ this.httpServer.setAttribute(JspHelper.CURRENT_CONF, this.conf);
+ setupServlets(this.httpServer, this.conf);
+
+ this.httpServer.start();
+
+ // The server port can be ephemeral... ensure we have the correct info
+ InetSocketAddress listenAddress = this.httpServer.getConnectorAddress(0);
+ if (listenAddress != null) {
+ this.httpAddress = new InetSocketAddress(this.httpAddress.getHostName(),
+ listenAddress.getPort());
+ }
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if(this.httpServer != null) {
+ this.httpServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ private static void setupServlets(
+ HttpServer2 httpServer, Configuration conf) {
+ // TODO Add servlets for FSCK, etc
+ }
+
+ public InetSocketAddress getHttpAddress() {
+ return this.httpAddress;
+ }
+
+ public InetSocketAddress getHttpsAddress() {
+ return this.httpsAddress;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index 5c33c2e..932295e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -35,11 +35,13 @@
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -713,6 +715,7 @@
* Re-throws exceptions generated by the remote RPC call as either
* RemoteException or IOException.
*
+ * @param <T> The type of the remote location.
* @param locations List of remote locations to call concurrently.
* @param remoteMethod The remote method and parameters to invoke.
* @param requireResponse If true an exception will be thrown if all calls do
@@ -723,10 +726,35 @@
* @throws IOException If requiredResponse=true and any of the calls throw an
* exception.
*/
- @SuppressWarnings("unchecked")
public <T extends RemoteLocationContext> Map<T, Object> invokeConcurrent(
final Collection<T> locations, final RemoteMethod method,
boolean requireResponse, boolean standby) throws IOException {
+ return invokeConcurrent(locations, method, requireResponse, standby, -1);
+ }
+
+ /**
+ * Invokes multiple concurrent proxy calls to different clients. Returns an
+ * array of results.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param locations List of remote locations to call concurrently.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @param requireResponse If true an exception will be thrown if all calls do
+ * not complete. If false exceptions are ignored and all data results
+ * successfully received are returned.
+ * @param standby If the requests should go to the standby namenodes too.
+ * @param timeoutMs Timeout for each individual call.
+ * @return Result of invoking the method per subcluster: nsId -> result.
+ * @throws IOException If requiredResponse=true and any of the calls throw an
+ * exception.
+ */
+ @SuppressWarnings("unchecked")
+ public <T extends RemoteLocationContext> Map<T, Object> invokeConcurrent(
+ final Collection<T> locations, final RemoteMethod method,
+ boolean requireResponse, boolean standby, long timeOutMs)
+ throws IOException {
final UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
final Method m = method.getMethod();
@@ -782,7 +810,13 @@
}
try {
- List<Future<Object>> futures = executorService.invokeAll(callables);
+ List<Future<Object>> futures = null;
+ if (timeOutMs > 0) {
+ futures = executorService.invokeAll(
+ callables, timeOutMs, TimeUnit.MILLISECONDS);
+ } else {
+ futures = executorService.invokeAll(callables);
+ }
Map<T, Object> results = new TreeMap<>();
Map<T, IOException> exceptions = new TreeMap<>();
for (int i=0; i<futures.size(); i++) {
@@ -791,6 +825,13 @@
Future<Object> future = futures.get(i);
Object result = future.get();
results.put(location, result);
+ } catch (CancellationException ce) {
+ T loc = orderedLocations.get(i);
+ String msg =
+ "Invocation to \"" + loc + "\" for \"" + method + "\" timed out";
+ LOG.error(msg);
+ IOException ioe = new IOException(msg);
+ exceptions.put(location, ioe);
} catch (ExecutionException ex) {
Throwable cause = ex.getCause();
LOG.debug("Canot execute {} in {}: {}",
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 650c6ab..4d3c237 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -1091,6 +1091,19 @@
public DatanodeInfo[] getDatanodeReport(DatanodeReportType type)
throws IOException {
checkOperation(OperationCategory.UNCHECKED);
+ return getDatanodeReport(type, 0);
+ }
+
+ /**
+ * Get the datanode report with a timeout.
+ * @param type Type of the datanode.
+ * @param timeOutMs Time out for the reply in milliseconds.
+ * @return List of datanodes.
+ * @throws IOException If it cannot get the report.
+ */
+ public DatanodeInfo[] getDatanodeReport(
+ DatanodeReportType type, long timeOutMs) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
Map<String, DatanodeInfo> datanodesMap = new LinkedHashMap<>();
RemoteMethod method = new RemoteMethod("getDatanodeReport",
@@ -1098,7 +1111,7 @@
Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
Map<FederationNamespaceInfo, Object> results =
- rpcClient.invokeConcurrent(nss, method, true, false);
+ rpcClient.invokeConcurrent(nss, method, true, false, timeOutMs);
for (Entry<FederationNamespaceInfo, Object> entry : results.entrySet()) {
FederationNamespaceInfo ns = entry.getKey();
DatanodeInfo[] result = (DatanodeInfo[]) entry.getValue();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index e1c5086..3491ed2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4802,6 +4802,62 @@
</property>
<property>
+ <name>dfs.federation.router.http-address</name>
+ <value>0.0.0.0:50071</value>
+ <description>
+ HTTP address that handles the web requests to the Router.
+ The value of this property will take the form of router-host1:http-port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.http-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the HTTP server will bind to. If this optional
+ address is set, it overrides only the hostname portion of
+ dfs.federation.router.http-address. This is useful for making the name
+ node listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.https-address</name>
+ <value>0.0.0.0:50072</value>
+ <description>
+ HTTPS address that handles the web requests to the Router.
+ The value of this property will take the form of router-host1:https-port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.https-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the HTTPS server will bind to. If this optional
+ address is set, it overrides only the hostname portion of
+ dfs.federation.router.https-address. This is useful for making the name
+ node listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.http.enable</name>
+ <value>true</value>
+ <description>
+ If the HTTP service to handle client requests in the router is enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.metrics.enable</name>
+ <value>true</value>
+ <description>
+ If the metrics service in the router is enabled.
+ </description>
+ </property>
+
+ <property>
<name>dfs.federation.router.file.resolver.client.class</name>
<value>org.apache.hadoop.hdfs.server.federation.MockResolver</value>
<description>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html
new file mode 100644
index 0000000..3da5283
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html
@@ -0,0 +1,371 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<meta http-equiv="X-UA-Compatible" content="IE=9" />
+<link rel="stylesheet" type="text/css" href="/static/bootstrap-3.0.2/css/bootstrap.min.css" />
+<link rel="stylesheet" type="text/css" href="/static/dataTables.bootstrap.css" />
+<link rel="stylesheet" type="text/css" href="/static/hadoop.css" />
+<title>Router Information</title>
+</head>
+<body>
+
+<header class="navbar navbar-inverse bs-docs-nav" role="banner">
+<div class="container">
+ <div class="navbar-header">
+ <div class="navbar-brand">Hadoop</div>
+ </div>
+
+ <ul class="nav navbar-nav" id="ui-tabs">
+ <li><a href="#tab-overview">Overview</a></li>
+ <li><a href="#tab-namenode">Subclusters</a></li>
+ <li><a href="#tab-datanode">Datanodes</a></li>
+ <li><a href="#tab-mounttable">Mount table</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Utilities <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+ <li><a href="jmx">Metrics</a></li>
+ <li><a href="conf">Configuration</a></li>
+ <li><a href="logs">Logs</a></li>
+ <li><a href="fsck">FSCK</a></li>
+ </ul>
+ </li>
+ </ul>
+</div>
+</header>
+
+<div class="container">
+
+<div id="alert-panel">
+ <div class="alert alert-danger">
+ <button type="button" class="close" onclick="$('#alert-panel').hide();">×</button>
+ <div class="alert-body" id="alert-panel-body"></div>
+ </div>
+</div>
+
+<div class="tab-content">
+ <div class="tab-pane" id="tab-overview"></div>
+ <div class="tab-pane" id="tab-namenode"></div>
+ <div class="tab-pane" id="tab-datanode"></div>
+ <div class="tab-pane" id="tab-mounttable"></div>
+</div>
+
+<div class="row">
+ <hr />
+ <div class="col-xs-2"><p>Hadoop, {release-year-token}.</p></div>
+</div>
+</div>
+
+<!-- Overview -->
+<script type="text/x-dust-template" id="tmpl-federationhealth">
+<div class="page-header"><h1>Router {#federation}<small>'{HostAndPort}'</small>{/federation}</h1></div>
+{#federation}
+<table class="table table-bordered table-striped">
+ <tr><th>Started:</th><td>{RouterStarted}</td></tr>
+ <tr><th>Version:</th><td>{Version}</td></tr>
+ <tr><th>Compiled:</th><td>{CompileInfo}</td></tr>
+ <tr><th>Cluster ID:</th><td>{ClusterId}</td></tr>
+ <tr><th>Block Pool ID:</th><td>{BlockPoolId}</td></tr>
+ <tr><th>Status:</th><td>{RouterStatus}</td></tr>
+</table>
+{/federation}
+
+<div class="page-header"><h1>Summary</h1></div>
+{#federation}
+<table class="table table-bordered table-striped">
+ <tr><th>Total capacity</th><td>{TotalCapacity|fmt_bytes}</td></tr>
+ <tr><th>Used capacity</th><td>{UsedCapacity|fmt_bytes}</td></tr>
+ <tr><th>Remaining capacity</th><td>{RemainingCapacity|fmt_bytes}</td></tr>
+ <tr><th>Nameservices</th><td>{NumNameservices}</td></tr>
+ <tr><th>Namenodes</th><td>{NumNamenodes}</td></tr>
+ <tr>
+ <th>DataNodes usages% (Min/Median/Max/stdDev)</th>
+ <td>{#NodeUsage.nodeUsage}{min} / {median} / {max} / {stdDev}{/NodeUsage.nodeUsage}</td>
+ </tr>
+ <tr><th><a href="#tab-datanode">Live Nodes</a></th><td>{NumLiveNodes} (Decommissioned: {NumDecomLiveNodes})</td></tr>
+ <tr><th><a href="#tab-datanode">Dead Nodes</a></th><td>{NumDeadNodes} (Decommissioned: {NumDecomDeadNodes})</td></tr>
+ <tr><th><a href="#tab-datanode">Decommissioning Nodes</a></th><td>{NumDecommissioningNodes}</td></tr>
+ <tr><th>Files</th><td>{NumFiles}</td></tr>
+ <tr><th>Blocks</th><td>{NumBlocks}</td></tr>
+ <tr><th title="Excludes missing blocks.">Number of Under-Replicated Blocks</th><td>{NumOfBlocksUnderReplicated}</td></tr>
+ <tr><th>Number of Blocks Pending Deletion</th><td>{NumOfBlocksPendingDeletion}</td></tr>
+</table>
+{/federation}
+</script>
+
+<!-- Subclusters info: list of nameservices and namenodes -->
+<script type="text/x-dust-template" id="tmpl-namenode">
+<div class="page-header"><h1>Nameservice Information</h1></div>
+<div>
+ <ul class="dfshealth-node-legend">
+ <li class="dfshealth-node-icon dfshealth-node-alive">Active</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Standby</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-maintenance">Safe mode</li>
+ <li class="dfshealth-node-icon dfshealth-node-down">Unavailable</li>
+ </ul>
+</div>
+<small>
+<table class="table">
+ <thead>
+ <tr>
+ <th colspan="6"></th>
+ <th colspan="3">Blocks</th>
+ <th colspan="2">Nodes</th>
+ <th colspan="3">Decom</th>
+ </tr>
+ <tr>
+ <th></th>
+ <th>Nameservice</th>
+ <th>Namenode</th>
+ <th>Last Contact</th>
+ <th>Capacity</th>
+ <th>Files</th>
+ <th>Total</th>
+ <th>Missing</th>
+ <th>Under-Replicated</th>
+ <th>Live</th>
+ <th>Dead</th>
+ <th>Progress</th>
+ <th>Live</th>
+ <th>Dead</th>
+ </tr>
+ </thead>
+ <tbody>
+ {#Nameservices}
+ <tr>
+ <td class="dfshealth-node-icon dfshealth-node-{iconState}" title="{title}"></td>
+ <td><a href="http://{webAddress}">{nameserviceId}</a></td>
+ <td><a href="http://{webAddress}">{namenodeId}</a></td>
+ <td>{lastHeartbeat}</td>
+ <td ng-value="{usedPercentage}" style="width:210px">
+ <div>
+ <div style="display:inline-block; float: left; padding-right: 10px; width:60px;">{totalSpace|fmt_bytes}</div>
+ <div class="clearfix progress dfshealth-node-capacity-bar" title="Used: {used|fmt_bytes}">
+ <div class="progress-bar {#helper_usage_bar value="{usedPercentage}"/}" style="width: {usedPercentage}%">
+ </div>
+ </div>
+ </div>
+ </td>
+ <td>{numOfFiles}</td>
+ <td>{numOfBlocks}</td>
+ <td>{numOfBlocksMissing}</td>
+ <td>{numOfBlocksUnderReplicated}</td>
+ <td>{numOfActiveDatanodes}</td>
+ <td>{numOfDeadDatanodes}</td>
+ <td>{numOfDecommissioningDatanodes}</td>
+ <td>{numOfDecomActiveDatanodes}</td>
+ <td>{numOfDecomDeadDatanodes}</td>
+ </tr>
+ {/Nameservices}
+ </tbody>
+</table>
+</small>
+
+<div class="page-header"><h1>Namenode Information</h1></div>
+<div>
+ <ul class="dfshealth-node-legend">
+ <li class="dfshealth-node-icon dfshealth-node-alive">Active</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Standby</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-maintenance">Safe mode</li>
+ <li class="dfshealth-node-icon dfshealth-node-down">Unavailable</li>
+ </ul>
+</div>
+<small>
+<table class="table">
+ <thead>
+ <tr>
+ <th colspan="7"></th>
+ <th colspan="3">Blocks</th>
+ <th colspan="2">Nodes</th>
+ <th colspan="3">Decom</th>
+ </tr>
+ <tr>
+ <th></th>
+ <th colspan="2">Namenode</th>
+ <th>Web address</th>
+ <th>Last Contact</th>
+ <th>Capacity</th>
+ <th>Files</th>
+ <th>Total</th>
+ <th>Missing</th>
+ <th>Under-Replicated</th>
+ <th>Live</th>
+ <th>Dead</th>
+ <th>Progress</th>
+ <th>Live</th>
+ <th>Dead</th>
+ </tr>
+ </thead>
+ <tbody>
+ {#Namenodes}
+ <tr>
+ <td class="dfshealth-node-icon dfshealth-node-{iconState}" title="{title}"></td>
+ <td>{nameserviceId}</td>
+ <td>{namenodeId}</td>
+ <td><a href="http://{webAddress}">{webAddress}</a></td>
+ <td>{lastHeartbeat}</td>
+ <td ng-value="{usedPercentage}" style="width:210px">
+ <div>
+ <div style="display:inline-block; float: left; padding-right: 10px; width:60px;">{totalSpace|fmt_bytes}</div>
+ <div class="clearfix progress dfshealth-node-capacity-bar" title="Used: {used|fmt_bytes}">
+ <div class="progress-bar {#helper_usage_bar value="{usedPercentage}"/}" style="width: {usedPercentage}%">
+ </div>
+ </div>
+ </div>
+ </td>
+ <td>{numOfFiles}</td>
+ <td>{numOfBlocks}</td>
+ <td>{numOfBlocksMissing}</td>
+ <td>{numOfBlocksUnderReplicated}</td>
+ <td>{numOfActiveDatanodes}</td>
+ <td>{numOfDeadDatanodes}</td>
+ <td>{numOfDecommissioningDatanodes}</td>
+ <td>{numOfDecomActiveDatanodes}</td>
+ <td>{numOfDecomDeadDatanodes}</td>
+ </tr>
+ {/Namenodes}
+ </tbody>
+</table>
+</small>
+</script>
+
+<!-- Datanodes -->
+<script type="text/x-dust-template" id="tmpl-datanode">
+<div class="page-header"><h1>Datanode Information</h1></div>
+<div>
+ <ul class="dfshealth-node-legend">
+ <li class="dfshealth-node-icon dfshealth-node-alive">In service</li>
+ <li class="dfshealth-node-icon dfshealth-node-down">Down</li>
+ <li class="dfshealth-node-icon dfshealth-node-decommisioned">Decommisioned</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-decommisioned">Decommissioned & dead</li>
+ </ul>
+</div>
+<div class="page-header"><h1><small>In operation</small></h1></div>
+<small>
+<table class="table" id="table-datanodes">
+ <thead>
+ <tr>
+ <th>Node</th>
+ <th>Last contact</th>
+ <th style="width:180px; text-align:center">Capacity</th>
+ <!--th>Blocks</th-->
+ <th>Block pool used</th>
+ <!--th>Version</th-->
+ </tr>
+ </thead>
+ {#LiveNodes}
+ <tr>
+ <td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td>
+ <td ng-value="{lastContact}">{#helper_relative_time value="{lastContact}"/}</td>
+ <td ng-value="{usedPercentage}" style="width:210px">
+ <div>
+ <div style="display:inline-block; float: left; padding-right: 10px; width:60px;">{capacity|fmt_bytes}</div>
+ <div class="clearfix progress dfshealth-node-capacity-bar" title="Non DFS: {nonDfsUsedSpace|fmt_bytes}, Used: {used|fmt_bytes}">
+ <div class="progress-bar {#helper_usage_bar value="{usedPercentage}"/}" style="width: {usedPercentage}%">
+ </div>
+ </div>
+ </div>
+ </td>
+ <!--td>{numBlocks}</td-->
+ <td ng-value="{blockPoolUsedPercent}">{blockPoolUsed|fmt_bytes} ({blockPoolUsedPercent|fmt_percentage})</td>
+ <!--td>{version}</td-->
+ </tr>
+ {/LiveNodes}
+ {#DeadNodes}
+ <tr class="danger">
+ <td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td>
+ <td>{#helper_relative_time value="{lastContact}"/}</td>
+ <td></td>
+ <!--td></td-->
+ <td></td>
+ <!--td></td-->
+ </tr>
+ {/DeadNodes}
+</table>
+</small>
+
+<div class="page-header"><h1><small>Decommissioning</small></h1></div>
+<small>
+<table class="table">
+ <thead>
+ <tr>
+ <th>Node</th>
+ <th>Under replicated blocks</th>
+ <th>Blocks with no live replicas</th>
+ <th>Under Replicated Blocks <br/>In files under construction</th>
+ </tr>
+ </thead>
+ {#DecomNodes}
+ <tr>
+ <td>{location}/{name} ({xferaddr})</td>
+ <td>{underReplicatedBlocks}</td>
+ <td>{decommissionOnlyReplicas}</td>
+ <td>{underReplicateInOpenFiles}</td>
+ </tr>
+ {/DecomNodes}
+</table>
+</small>
+</script>
+
+<!-- Mount table -->
+<script type="text/x-dust-template" id="tmpl-mounttable">
+<div class="page-header"><h1>Mount Table</h1></div>
+<small>
+<table class="table">
+ <thead>
+ <tr>
+ <th>Global path</th>
+ <th>Target nameservice</th>
+ <th>Target path</th>
+ <th>Order</th>
+ <th>Read only</th>
+ <th>Date Modified</th>
+ <th>Date Created</th>
+ </tr>
+ </thead>
+ <tbody>
+ {#MountTable}
+ <tr>
+ <td>{sourcePath}</td>
+ <td>{nameserviceId}</td>
+ <td>{path}</td>
+ <td>{order}</td>
+ <td class="dfshealth-node-icon dfshealth-mount-read-only-{readonly}"/>
+ <td>{dateModified}</td>
+ <td>{dateCreated}</td>
+ </tr>
+ {/MountTable}
+ </tbody>
+</table>
+</small>
+</script>
+
+
+
+<script type="text/javascript" src="/static/jquery-1.10.2.min.js"></script>
+<script type="text/javascript" src="/static/jquery.dataTables.min.js"></script>
+<script type="text/javascript" src="/static/bootstrap-3.0.2/js/bootstrap.min.js"></script>
+<script type="text/javascript" src="/static/dataTables.bootstrap.js"></script>
+<script type="text/javascript" src="/static/moment.min.js"></script>
+<script type="text/javascript" src="/static/dust-full-2.0.0.min.js"></script>
+<script type="text/javascript" src="/static/dust-helpers-1.1.1.min.js"></script>
+<script type="text/javascript" src="/static/dfs-dust.js"></script>
+<script type="text/javascript" src="federationhealth.js"></script>
+</body>
+</html>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.js
new file mode 100644
index 0000000..380f097
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.js
@@ -0,0 +1,313 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+(function () {
+ "use strict";
+
+ dust.loadSource(dust.compile($('#tmpl-federationhealth').html(), 'federationhealth'));
+ dust.loadSource(dust.compile($('#tmpl-namenode').html(), 'namenode-info'));
+ dust.loadSource(dust.compile($('#tmpl-datanode').html(), 'datanode-info'));
+ dust.loadSource(dust.compile($('#tmpl-mounttable').html(), 'mounttable'));
+
+ $.fn.dataTable.ext.order['ng-value'] = function (settings, col)
+ {
+ return this.api().column(col, {order:'index'} ).nodes().map(function (td, i) {
+ return $(td).attr('ng-value');
+ });
+ };
+
+ function load_overview() {
+ var BEANS = [
+ {"name": "federation", "url": "/jmx?qry=Hadoop:service=Router,name=FederationState"}
+ ];
+
+ var HELPERS = {
+ 'helper_fs_max_objects': function (chunk, ctx, bodies, params) {
+ var o = ctx.current();
+ if (o.MaxObjects > 0) {
+ chunk.write('(' + Math.round((o.FilesTotal + o.BlockTotal) / o.MaxObjects * 100) * 100 + ')%');
+ }
+ },
+
+ 'helper_dir_status': function (chunk, ctx, bodies, params) {
+ var j = ctx.current();
+ for (var i in j) {
+ chunk.write('<tr><td>' + i + '</td><td>' + j[i] + '</td><td>' + params.type + '</td></tr>');
+ }
+ },
+
+ 'helper_date_tostring' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ return chunk.write('' + new Date(Number(value)).toLocaleString());
+ }
+ };
+
+ var data = {};
+
+ // Workarounds for the fact that JMXJsonServlet returns non-standard JSON strings
+ function workaround(nn) {
+ nn.NodeUsage = JSON.parse(nn.NodeUsage);
+ return nn;
+ }
+
+ load_json(
+ BEANS,
+ guard_with_startup_progress(function(d) {
+ for (var k in d) {
+ data[k] = k === 'federation' ? workaround(d[k].beans[0]) : d[k].beans[0];
+ }
+ render();
+ }),
+ function (url, jqxhr, text, err) {
+ show_err_msg('<p>Failed to retrieve data from ' + url + ', cause: ' + err + '</p>');
+ });
+
+ function render() {
+ var base = dust.makeBase(HELPERS);
+ dust.render('federationhealth', base.push(data), function(err, out) {
+ $('#tab-overview').html(out);
+ $('#ui-tabs a[href="#tab-overview"]').tab('show');
+ });
+ }
+ }
+
+ function load_namenode_info() {
+ var HELPERS = {
+ 'helper_lastcontact_tostring' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ return chunk.write('' + new Date(Date.now()-1000*Number(value)));
+ }
+ };
+
+ function workaround(r) {
+ function node_map_to_array(nodes) {
+ var res = [];
+ for (var n in nodes) {
+ var p = nodes[n];
+ p.name = n;
+ res.push(p);
+ }
+ return res;
+ }
+
+ function capitalise(string) {
+ return string.charAt(0).toUpperCase() + string.slice(1).toLowerCase();
+ }
+
+ function augment_namenodes(nodes) {
+ for (var i = 0, e = nodes.length; i < e; ++i) {
+ var n = nodes[i];
+ n.usedPercentage = Math.round(n.used * 1.0 / n.totalSpace * 100);
+ n.title = "Unavailable";
+ n.iconState = "down";
+ if (n.isSafeMode === true) {
+ n.title = capitalise(n.state) + " (safe mode)"
+ n.iconState = "decommisioned";
+ } else if (n.state === "ACTIVE") {
+ n.title = capitalise(n.state);
+ n.iconState = "alive";
+ } else if (nodes[i].state === "STANDBY") {
+ n.title = capitalise(n.state);
+ n.iconState = "down-decommisioned";
+ } else if (nodes[i].state === "UNAVAILABLE") {
+ n.title = capitalise(n.state);
+ n.iconState = "down";
+ }
+ if (n.namenodeId === "null") {
+ n.namenodeId = "";
+ }
+ }
+ }
+
+ r.Nameservices = node_map_to_array(JSON.parse(r.Nameservices));
+ augment_namenodes(r.Nameservices);
+ r.Namenodes = node_map_to_array(JSON.parse(r.Namenodes));
+ augment_namenodes(r.Namenodes);
+ return r;
+ }
+
+ $.get(
+ '/jmx?qry=Hadoop:service=Router,name=FederationState',
+ guard_with_startup_progress(function (resp) {
+ var data = workaround(resp.beans[0]);
+ var base = dust.makeBase(HELPERS);
+ dust.render('namenode-info', base.push(data), function(err, out) {
+ $('#tab-namenode').html(out);
+ $('#ui-tabs a[href="#tab-namenode"]').tab('show');
+ });
+ })).error(ajax_error_handler);
+ }
+
+ // TODO Copied directly from dfshealth.js; is there a way to import this function?
+ function load_datanode_info() {
+
+ var HELPERS = {
+ 'helper_relative_time' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ return chunk.write(moment().subtract(Number(value), 'seconds').format('YYYY-MM-DD HH:mm:ss'));
+ },
+ 'helper_usage_bar' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ var v = Number(value);
+ var r = null;
+ if (v < 70) {
+ r = 'progress-bar-success';
+ } else if (v < 85) {
+ r = 'progress-bar-warning';
+ } else {
+ r = "progress-bar-danger";
+ }
+ return chunk.write(r);
+ },
+ };
+
+ function workaround(r) {
+ function node_map_to_array(nodes) {
+ var res = [];
+ for (var n in nodes) {
+ var p = nodes[n];
+ p.name = n;
+ res.push(p);
+ }
+ return res;
+ }
+
+ function augment_live_nodes(nodes) {
+ for (var i = 0, e = nodes.length; i < e; ++i) {
+ var n = nodes[i];
+ n.usedPercentage = Math.round((n.used + n.nonDfsUsedSpace) * 1.0 / n.capacity * 100);
+ if (n.adminState === "In Service") {
+ n.state = "alive";
+ } else if (nodes[i].adminState === "Decommission In Progress") {
+ n.state = "decommisioning";
+ } else if (nodes[i].adminState === "Decommissioned") {
+ n.state = "decommissioned";
+ }
+ }
+ }
+
+ function augment_dead_nodes(nodes) {
+ for (var i = 0, e = nodes.length; i < e; ++i) {
+ if (nodes[i].decommissioned) {
+ nodes[i].state = "down-decommissioned";
+ } else {
+ nodes[i].state = "down";
+ }
+ }
+ }
+
+ r.LiveNodes = node_map_to_array(JSON.parse(r.LiveNodes));
+ augment_live_nodes(r.LiveNodes);
+ r.DeadNodes = node_map_to_array(JSON.parse(r.DeadNodes));
+ augment_dead_nodes(r.DeadNodes);
+ r.DecomNodes = node_map_to_array(JSON.parse(r.DecomNodes));
+ return r;
+ }
+
+ $.get(
+ '/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo',
+ guard_with_startup_progress(function (resp) {
+ var data = workaround(resp.beans[0]);
+ var base = dust.makeBase(HELPERS);
+ dust.render('datanode-info', base.push(data), function(err, out) {
+ $('#tab-datanode').html(out);
+ $('#table-datanodes').dataTable( {
+ 'lengthMenu': [ [25, 50, 100, -1], [25, 50, 100, "All"] ],
+ 'columns': [
+ { 'orderDataType': 'ng-value', 'searchable': true },
+ { 'orderDataType': 'ng-value', 'type': 'numeric' },
+ { 'orderDataType': 'ng-value', 'type': 'numeric' },
+ { 'orderDataType': 'ng-value', 'type': 'numeric'}
+ ]});
+ $('#ui-tabs a[href="#tab-datanode"]').tab('show');
+ });
+ })).error(ajax_error_handler);
+ }
+
+ function load_mount_table() {
+ var HELPERS = {}
+
+ function workaround(resource) {
+ resource.MountTable = JSON.parse(resource.MountTable)
+ return resource;
+ }
+
+ $.get(
+ '/jmx?qry=Hadoop:service=Router,name=FederationState',
+ guard_with_startup_progress(function (resp) {
+ var data = workaround(resp.beans[0]);
+ var base = dust.makeBase(HELPERS);
+ dust.render('mounttable', base.push(data), function(err, out) {
+ $('#tab-mounttable').html(out);
+ $('#ui-tabs a[href="#tab-mounttable"]').tab('show');
+ });
+ })).error(ajax_error_handler);
+ }
+
+ function toTitleCase(str) {
+ return str.replace(/\w\S*/g, function(txt){
+ return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase();
+ });
+ }
+
+ function show_err_msg(msg) {
+ $('#alert-panel-body').html(msg);
+ $('#alert-panel').show();
+ }
+
+ function ajax_error_handler(url, jqxhr, text, err) {
+ show_err_msg('<p>Failed to retrieve data from ' + url + ', cause: ' + err + '</p>');
+ }
+
+ function guard_with_startup_progress(fn) {
+ return function() {
+ try {
+ fn.apply(this, arguments);
+ } catch (err) {
+ if (err instanceof TypeError) {
+ show_err_msg('Router error: ' + err);
+ }
+ }
+ };
+ }
+
+ function load_page() {
+ var hash = window.location.hash;
+ switch(hash) {
+ case "#tab-mounttable":
+ load_mount_table();
+ break;
+ case "#tab-namenode":
+ load_namenode_info();
+ break;
+ case "#tab-datanode":
+ load_datanode_info();
+ break;
+ case "#tab-overview":
+ load_overview();
+ break;
+ default:
+ window.location.hash = "tab-overview";
+ break;
+ }
+ }
+ load_page();
+
+ $(window).bind('hashchange', function () {
+ load_page();
+ });
+})();
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/index.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/index.html
new file mode 100644
index 0000000..4ee04ac
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/index.html
@@ -0,0 +1,24 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<meta http-equiv="REFRESH" content="0;url=federationhealth.html" />
+<title>Hadoop Administration</title>
+</head>
+</html>
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
index 58ca1d1..88da77e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
@@ -29,6 +29,7 @@
private boolean enableRpcServer = false;
private boolean enableAdminServer = false;
+ private boolean enableHttpServer = false;
private boolean enableHeartbeat = false;
private boolean enableLocalHeartbeat = false;
private boolean enableStateStore = false;
@@ -45,6 +46,7 @@
public RouterConfigBuilder all() {
this.enableRpcServer = true;
this.enableAdminServer = true;
+ this.enableHttpServer = true;
this.enableHeartbeat = true;
this.enableLocalHeartbeat = true;
this.enableStateStore = true;
@@ -67,6 +69,11 @@
return this;
}
+ public RouterConfigBuilder http(boolean enable) {
+ this.enableHttpServer = enable;
+ return this;
+ }
+
public RouterConfigBuilder heartbeat(boolean enable) {
this.enableHeartbeat = enable;
return this;
@@ -90,6 +97,10 @@
return this.admin(true);
}
+ public RouterConfigBuilder http() {
+ return this.http(true);
+ }
+
public RouterConfigBuilder heartbeat() {
return this.heartbeat(true);
}
@@ -108,6 +119,8 @@
conf.setBoolean(DFSConfigKeys.DFS_ROUTER_RPC_ENABLE, this.enableRpcServer);
conf.setBoolean(DFSConfigKeys.DFS_ROUTER_ADMIN_ENABLE,
this.enableAdminServer);
+ conf.setBoolean(DFSConfigKeys.DFS_ROUTER_HTTP_ENABLE,
+ this.enableHttpServer);
conf.setBoolean(DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
this.enableHeartbeat);
conf.setBoolean(DFSConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
index e79674d..4ea0f19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
@@ -33,6 +33,9 @@
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HTTP_BIND_HOST_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_MONITOR_NAMENODE;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY;
@@ -134,6 +137,7 @@
private String nameserviceId;
private String namenodeId;
private int rpcPort;
+ private int httpPort;
private DFSClient client;
private Configuration conf;
private RouterClient adminClient;
@@ -164,6 +168,10 @@
return this.rpcPort;
}
+ public int getHttpPort() {
+ return this.httpPort;
+ }
+
public FileContext getFileContext() {
return this.fileContext;
}
@@ -183,6 +191,10 @@
this.fileContext = null;
}
}
+ InetSocketAddress httpAddress = router.getHttpServerAddress();
+ if (httpAddress != null) {
+ this.httpPort = httpAddress.getPort();
+ }
}
public FileSystem getFileSystem() throws IOException {
@@ -399,14 +411,19 @@
conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix,
"127.0.0.1:" + context.rpcPort);
- conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + "." + suffix,
- "127.0.0.1:" + context.servicePort);
conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix,
"127.0.0.1:" + context.httpPort);
conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + suffix,
"0.0.0.0");
- conf.set(DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY + "." + suffix,
- "0.0.0.0");
+
+ // If the service port is enabled by default, we need to set them up
+ boolean servicePortEnabled = false;
+ if (servicePortEnabled) {
+ conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + "." + suffix,
+ "127.0.0.1:" + context.servicePort);
+ conf.set(DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY + "." + suffix,
+ "0.0.0.0");
+ }
}
}
@@ -447,6 +464,10 @@
conf.set(DFS_ROUTER_ADMIN_ADDRESS_KEY, "127.0.0.1:0");
conf.set(DFS_ROUTER_ADMIN_BIND_HOST_KEY, "0.0.0.0");
+ conf.set(DFS_ROUTER_HTTP_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_ROUTER_HTTPS_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_ROUTER_HTTP_BIND_HOST_KEY, "0.0.0.0");
+
conf.set(DFS_ROUTER_DEFAULT_NAMESERVICE, nameservices.get(0));
conf.setLong(DFS_ROUTER_HEARTBEAT_INTERVAL_MS, heartbeatInterval);
conf.setLong(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, cacheFlushInterval);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java
index bbcfbe8..b455177 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java
@@ -64,6 +64,7 @@
routerConfig = new RouterConfigBuilder()
.stateStore()
.metrics()
+ .http()
.build();
router = new Router();
router.init(routerConfig);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
index e22be84..77d1698 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
@@ -24,6 +24,7 @@
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.federation.MockResolver;
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
@@ -61,10 +62,13 @@
conf.set(DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
conf.set(DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, "127.0.0.1:0");
conf.set(DFSConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY, "0.0.0.0");
+ conf.set(DFSConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFSConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFSConfigKeys.DFS_ROUTER_HTTP_BIND_HOST_KEY, "0.0.0.0");
// Simulate a co-located NN
conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns0");
- conf.set("fs.defaultFS", "hdfs://" + "ns0");
+ conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + "ns0");
conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + "ns0",
"127.0.0.1:0" + 0);
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + "ns0",
@@ -104,6 +108,9 @@
// Admin only
testRouterStartup(new RouterConfigBuilder(conf).admin().build());
+ // Http only
+ testRouterStartup(new RouterConfigBuilder(conf).http().build());
+
// Rpc only
testRouterStartup(new RouterConfigBuilder(conf).rpc().build());
