MAPREDUCE-1989. Fixes error message in gridmix when user resolver is set and no user list is given. Contributed by Ravi Gummadi
git-svn-id: https://svn.apache.org/repos/asf/hadoop/mapreduce/trunk@1001603 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 8db7e83..0ba2371 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -304,6 +304,9 @@
MAPREDUCE-2078. Fixes TraceBuilder to generate traces when a globbed job
history path is given. (Amar Kamat via amareshwari)
+ MAPREDUCE-1989. Fixes error message in gridmix when user resolver is set
+ and no user list is given. (Ravi Gummadi via amareshwari)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
diff --git a/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/EchoUserResolver.java b/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/EchoUserResolver.java
index a1bc2f5..2fcb39d 100644
--- a/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/EchoUserResolver.java
+++ b/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/EchoUserResolver.java
@@ -19,15 +19,9 @@
import java.io.IOException;
import java.net.URI;
-import java.util.Collections;
-import java.util.List;
-import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
-import org.apache.hadoop.security.Groups;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -50,4 +44,14 @@
UserGroupInformation ugi) {
return ugi;
}
+
+ /**
+ * {@inheritDoc}
+ * <br><br>
+ * Since {@link EchoUserResolver} simply returns the user's name passed as
+ * the argument, it doesn't need a target list of users.
+ */
+ public boolean needsTargetUsersList() {
+ return false;
+ }
}
diff --git a/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java b/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
index 63baaf3..afb1d21 100644
--- a/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
+++ b/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
@@ -252,9 +252,22 @@
return 1;
}
}
- if (!userResolver.setTargetUsers(userRsrc, conf)) {
- LOG.warn("Resource " + userRsrc + " ignored");
+
+ if (userResolver.needsTargetUsersList()) {
+ if (userRsrc != null) {
+ if (!userResolver.setTargetUsers(userRsrc, conf)) {
+ LOG.warn("Ignoring the user resource '" + userRsrc + "'.");
+ }
+ } else {
+ System.err.println("\n\n" + userResolver.getClass()
+ + " needs target user list. Use -users option." + "\n\n");
+ printUsage(System.err);
+ return 1;
+ }
+ } else if (userRsrc != null) {
+ LOG.warn("Ignoring the user resource '" + userRsrc + "'.");
}
+
ioPath = new Path(argv[argv.length - 2]);
traceIn = argv[argv.length - 1];
} catch (Exception e) {
diff --git a/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java b/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
index 5c844a5..813e047 100644
--- a/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
+++ b/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
@@ -91,12 +91,17 @@
throws IOException {
users = parseUserList(userloc, conf);
if (users.size() == 0) {
- throw new IOException("Empty user list");
+ throw new IOException(buildEmptyUsersErrorMsg(userloc));
}
usercache.keySet().retainAll(users);
return true;
}
+ static String buildEmptyUsersErrorMsg(URI userloc) {
+ return "Empty user list is not allowed for RoundRobinUserResolver. Provided"
+ + " user resource URI '" + userloc + "' resulted in an empty user list.";
+ }
+
@Override
public synchronized UserGroupInformation getTargetUgi(
UserGroupInformation ugi) {
@@ -115,4 +120,14 @@
}
return val;
}
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * {@link RoundRobinUserResolver} needs to map the users in the
+ * trace to the provided list of target users. So user list is needed.
+ */
+ public boolean needsTargetUsersList() {
+ return true;
+ }
}
\ No newline at end of file
diff --git a/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/SubmitterUserResolver.java b/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/SubmitterUserResolver.java
index 050f31f..d0d552a 100644
--- a/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/SubmitterUserResolver.java
+++ b/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/SubmitterUserResolver.java
@@ -32,13 +32,13 @@
private UserGroupInformation ugi = null;
- public SubmitterUserResolver() {
+ public SubmitterUserResolver() throws IOException {
LOG.info(" Current user resolver is SubmitterUserResolver ");
+ ugi = UserGroupInformation.getLoginUser();
}
public synchronized boolean setTargetUsers(URI userdesc, Configuration conf)
throws IOException {
- ugi = UserGroupInformation.getLoginUser();
return false;
}
@@ -47,4 +47,13 @@
return this.ugi;
}
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Since {@link SubmitterUserResolver} returns the user name who is running
+ * gridmix, it doesn't need a target list of users.
+ */
+ public boolean needsTargetUsersList() {
+ return false;
+ }
}
diff --git a/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/UserResolver.java b/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/UserResolver.java
index a3c522e..ca8c98b 100644
--- a/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/UserResolver.java
+++ b/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/UserResolver.java
@@ -19,29 +19,27 @@
import java.io.IOException;
import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.LineReader;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
/**
* Maps users in the trace to a set of valid target users on the test cluster.
*/
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
public interface UserResolver {
/**
* Configure the user map given the URI and configuration. The resolver's
* contract will define how the resource will be interpreted, but the default
* will typically interpret the URI as a {@link org.apache.hadoop.fs.Path}
- * listing target users.
- * @param userdesc URI (possibly null) from which user information may be
- * loaded per the subclass contract.
+ * listing target users.
+ * This method should be called only if {@link #needsTargetUsersList()}
+ * returns true.
+ * @param userdesc URI from which user information may be loaded per the
+ * subclass contract.
* @param conf The tool configuration.
* @return true if the resource provided was used in building the list of
* target users
@@ -55,4 +53,13 @@
*/
public UserGroupInformation getTargetUgi(UserGroupInformation ugi);
+ /**
+ * Indicates whether this user resolver needs a list of target users to be
+ * provided.
+ *
+ * @return true if a list of target users is to be provided for this
+ * user resolver
+ */
+ public boolean needsTargetUsersList();
+
}
diff --git a/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java b/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
index 8229f88..97c6d81 100644
--- a/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
+++ b/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
@@ -28,29 +28,35 @@
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
public class TestUserResolve {
- static Path userlist;
+ private static Path rootDir = null;
+ private static Configuration conf = null;
+ private static FileSystem fs = null;
@BeforeClass
- public static void writeUserList() throws IOException {
- final Configuration conf = new Configuration();
- final FileSystem fs = FileSystem.getLocal(conf);
- final Path wd =
- new Path(new Path(System.getProperty("test.build.data", "/tmp"))
- .makeQualified(fs),
- "gridmixUserResolve");
- userlist = new Path(wd, "users");
+ public static void createRootDir() throws IOException {
+ conf = new Configuration();
+ fs = FileSystem.getLocal(conf);
+ rootDir = new Path(new Path(System.getProperty("test.build.data", "/tmp"))
+ .makeQualified(fs), "gridmixUserResolve");
+ }
+
+ /**
+ * Creates users file with the content as the String usersFileContent.
+ * @param usersFilePath the path to the file that is to be created
+ * @param usersFileContent Content of users file
+ * @throws IOException
+ */
+ private static void writeUserList(Path usersFilePath, String usersFileContent)
+ throws IOException {
+
FSDataOutputStream out = null;
try {
- out = fs.create(userlist, true);
- out.writeBytes("user0,groupA,groupB,groupC\n");
- out.writeBytes("user1,groupA,groupC\n");
- out.writeBytes("user2,groupB\n");
- out.writeBytes("user3,groupA,groupB,groupC\n");
+ out = fs.create(usersFilePath, true);
+ out.writeBytes(usersFileContent);
} finally {
if (out != null) {
out.close();
@@ -58,20 +64,65 @@
}
}
- @Test
- public void testRoundRobinResolver() throws Exception {
- final Configuration conf = new Configuration();
- final UserResolver rslv = new RoundRobinUserResolver();
-
+ /**
+ * Validate RoundRobinUserResolver's behavior for bad user resource file.
+ * RoundRobinUserResolver.setTargetUsers() should throw proper Exception for
+ * the cases like
+ * <li> non existent user resource file and
+ * <li> empty user resource file
+ *
+ * @param rslv The RoundRobinUserResolver object
+ * @param userRsrc users file
+ * @param expectedErrorMsg expected error message
+ */
+ private void validateBadUsersFile(UserResolver rslv, URI userRsrc,
+ String expectedErrorMsg) {
boolean fail = false;
try {
- rslv.setTargetUsers(null, conf);
+ rslv.setTargetUsers(userRsrc, conf);
} catch (IOException e) {
+ assertTrue("Exception message from RoundRobinUserResolver is wrong",
+ e.getMessage().equals(expectedErrorMsg));
fail = true;
}
assertTrue("User list required for RoundRobinUserResolver", fail);
+ }
- rslv.setTargetUsers(new URI(userlist.toString()), conf);
+ /**
+ * Validate the behavior of {@link RoundRobinUserResolver} for different
+ * user resource files like
+ * <li> Empty user resource file
+ * <li> Non existent user resource file
+ * <li> User resource file with valid content
+ * @throws Exception
+ */
+ @Test
+ public void testRoundRobinResolver() throws Exception {
+
+ final UserResolver rslv = new RoundRobinUserResolver();
+ Path usersFilePath = new Path(rootDir, "users");
+ URI userRsrc = new URI(usersFilePath.toString());
+
+ // Check if the error message is as expected for non existent
+ // user resource file.
+ fs.delete(usersFilePath, false);
+ String expectedErrorMsg = "File " + userRsrc + " does not exist.";
+ validateBadUsersFile(rslv, userRsrc, expectedErrorMsg);
+
+ // Check if the error message is as expected for empty user resource file
+ writeUserList(usersFilePath, "");// creates empty users file
+ expectedErrorMsg =
+ RoundRobinUserResolver.buildEmptyUsersErrorMsg(userRsrc);
+ validateBadUsersFile(rslv, userRsrc, expectedErrorMsg);
+
+ // Create user resource file with valid content
+ writeUserList(usersFilePath,
+ "user0,groupA,groupB,groupC\nuser1,groupA,groupC\n"
+ + "user2,groupB\nuser3,groupA,groupB,groupC\n");
+
+ // Validate RoundRobinUserResolver for the case of
+ // user resource file with valid content.
+ assertTrue(rslv.setTargetUsers(new URI(usersFilePath.toString()), conf));
UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser("hfre0");
assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
assertEquals("user1",
@@ -89,13 +140,9 @@
@Test
public void testSubmitterResolver() throws Exception {
- final Configuration conf = new Configuration();
final UserResolver rslv = new SubmitterUserResolver();
- rslv.setTargetUsers(null, conf);
+ assertFalse(rslv.needsTargetUsersList());
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
assertEquals(ugi, rslv.getTargetUgi((UserGroupInformation)null));
- System.out.println(" Submitter current user " + ugi);
- System.out.println(" Target ugi "
- + rslv.getTargetUgi((UserGroupInformation) null));
}
}
