MAPREDUCE-6519 Avoid unsafe split and append on fields that might be IPv6 literals
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index 2c58ebe..65dbdd2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StopWatch;
@@ -685,19 +686,19 @@ else if (index == endIndex) {
private String[] identifyHosts(int replicationFactor,
Map<Node,NodeInfo> racksMap) {
-
+
String [] retVal = new String[replicationFactor];
-
- List <NodeInfo> rackList = new LinkedList<NodeInfo>();
+
+ List <NodeInfo> rackList = new LinkedList<NodeInfo>();
rackList.addAll(racksMap.values());
-
+
// Sort the racks based on their contribution to this split
sortInDescendingOrder(rackList);
-
+
boolean done = false;
int index = 0;
-
+
// Get the host list for all our aggregated items, sort
// them and return the top entries
for (NodeInfo ni: rackList) {
@@ -706,27 +707,27 @@ else if (index == endIndex) {
List<NodeInfo>hostList = new LinkedList<NodeInfo>();
hostList.addAll(hostSet);
-
+
// Sort the hosts in this rack based on their contribution
sortInDescendingOrder(hostList);
for (NodeInfo host: hostList) {
// Strip out the port number from the host name
- retVal[index++] = host.node.getName().split(":")[0];
+ retVal[index++] = NetUtils.getHostFromHostPort(host.node.getName());
if (index == replicationFactor) {
done = true;
break;
}
}
-
+
if (done == true) {
break;
}
}
return retVal;
}
-
- private String[] fakeRacks(BlockLocation[] blkLocations, int index)
+
+ private String[] fakeRacks(BlockLocation[] blkLocations, int index)
throws IOException {
String[] allHosts = blkLocations[index].getHosts();
String[] allTopos = new String[allHosts.length];
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
index ad279ee..1ba4387 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.net.NetUtils;
@Private
@Unstable
@@ -56,10 +57,7 @@ public static String getTaskLogUrl(String taskTrackerHostName,
public static String convertTrackerNameToHostName(String trackerName) {
// Ugly!
// Convert the trackerName to its host name
- int indexOfColon = trackerName.indexOf(":");
- String trackerHostName = (indexOfColon == -1) ?
- trackerName :
- trackerName.substring(0, indexOfColon);
+ String trackerHostName = NetUtils.getHostFromHostPort(trackerName);
return trackerHostName.substring("tracker_".length());
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
index 3751ad9..1d5ed5f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
@@ -90,6 +90,7 @@
import org.apache.hadoop.yarn.webapp.WebApps;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.HostAndPort;
/**
* This module is responsible for talking to the
@@ -161,10 +162,12 @@ protected void initializeWebApp(Configuration conf) {
.withHttpSpnegoPrincipalKey(
JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY)
.at(NetUtils.getHostPortString(bindAddress)).start(webApp);
-
- String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0];
- MRWebAppUtil.setJHSWebappURLWithoutScheme(conf,
- connectHost + ":" + webApp.getListenerAddress().getPort());
+
+ String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf);
+
+ MRWebAppUtil.setJHSWebappURLWithoutScheme(conf, HostAndPort.fromParts(
+ HostAndPort.fromString(connectHost).getHostText(),
+ webApp.getListenerAddress().getPort()).toString());
}
@Override
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
index 0274298..dc0b09a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
@@ -36,6 +36,8 @@
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.net.HostAndPort;
+
/**
* This class checks that RPCs can use specialized socket factories.
*/
@@ -88,9 +90,9 @@ public void testSocketFactory() throws IOException {
"org.apache.hadoop.ipc.DummySocketFactory");
jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
String rmAddress = jconf.get("yarn.resourcemanager.address");
- String[] split = rmAddress.split(":");
- jconf.set("yarn.resourcemanager.address", split[0] + ':'
- + (Integer.parseInt(split[1]) + 10));
+ HostAndPort hp = HostAndPort.fromString(rmAddress);
+ jconf.set("yarn.resourcemanager.address", hp.getHostText() + ':'
+ + (hp.getPort() + 10));
client = new JobClient(jconf);
JobStatus[] jobs = client.jobsToComplete();
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
index e6e12eb..1d1ca2a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
@@ -35,6 +35,7 @@
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.util.HostUtil;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Shell;
@@ -340,7 +341,7 @@ private void stopTaskTrackers(ClusterStatus c) throws Exception {
LOG.info(new Date() + " Stopping a few trackers");
for (String tracker : trackerNamesList) {
- String host = convertTrackerNameToHostName(tracker);
+ String host = HostUtil.convertTrackerNameToHostName(tracker);
LOG.info(new Date() + " Marking tracker on host: " + host);
fos.write((host + "\n").getBytes());
if (count++ >= trackerNamesList.size()/2) {
@@ -378,16 +379,6 @@ private void runOperationOnTT(String operation) throws IOException {
LOG.info(output);
}
}
-
- private String convertTrackerNameToHostName(String trackerName) {
- // Convert the trackerName to it's host name
- int indexOfColon = trackerName.indexOf(":");
- String trackerHostName = (indexOfColon == -1) ?
- trackerName :
- trackerName.substring(0, indexOfColon);
- return trackerHostName.substring("tracker_".length());
- }
-
}
private class KillTaskThread extends Thread {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
index 8ab3304..51b030c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
@@ -133,6 +133,8 @@
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.net.HostAndPort;
+
public class TestClientRedirect {
static {
@@ -306,9 +308,9 @@ public GetApplicationReportResponse getApplicationReport(
application.setYarnApplicationState(YarnApplicationState.FINISHED);
application.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
}
- String[] split = AMHOSTADDRESS.split(":");
- application.setHost(split[0]);
- application.setRpcPort(Integer.parseInt(split[1]));
+ HostAndPort hp = HostAndPort.fromString(AMHOSTADDRESS);
+ application.setHost(hp.getHostText());
+ application.setRpcPort(hp.getPort());
application.setUser("TestClientRedirect-user");
application.setName("N/A");
application.setQueue("N/A");
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
index 972391c..3e7a596 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
@@ -797,20 +797,4 @@ static void setUpConfigFile(Properties confProps, File configFile)
file.close();
return file;
}
-
- /**
- * This formats the long tasktracker name to just the FQDN
- * @param taskTrackerLong String The long format of the tasktracker string
- * @return String The FQDN of the tasktracker
- * @throws Exception
- */
- public static String getFQDNofTT (String taskTrackerLong) throws Exception {
- //Getting the exact FQDN of the tasktracker from the tasktracker string.
- String[] firstSplit = taskTrackerLong.split("_");
- String tmpOutput = firstSplit[1];
- String[] secondSplit = tmpOutput.split(":");
- String tmpTaskTracker = secondSplit[0];
- return tmpTaskTracker;
- }
-
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
index 2e8ba5e..cae01e7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
@@ -47,6 +47,8 @@
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.mortbay.util.ajax.JSON;
+import com.google.common.net.HostAndPort;
+
/**
* This class drives the creation of a mini-cluster on the local machine. By
* default, a MiniDFSCluster and MiniMRCluster are spawned on the first
@@ -188,8 +190,8 @@ public void start() throws IOException, FileNotFoundException,
map.put("namenode_port", dfs.getNameNodePort());
}
if (mr != null) {
- map.put("resourcemanager_port", mr.getConfig().get(
- YarnConfiguration.RM_ADDRESS).split(":")[1]);
+ map.put("resourcemanager_port", HostAndPort.fromString(
+ mr.getConfig().get(YarnConfiguration.RM_ADDRESS)).getPort());
}
FileWriter fw = new FileWriter(new File(writeDetails));
fw.write(new JSON().toJSON(map));
