MAPREDUCE-871. Fix ownership of Job/Task local files to have correct group ownership according to the egid of the tasktracker. Contributed by Vinod Kumar Vavilapalli.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/mapreduce/trunk@808351 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 7541652..447737b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -497,3 +497,6 @@
MAPREDUCE-430. Fix a bug related to task getting stuck in case of
OOM error. (Amar Kamat via ddas)
+ MAPREDUCE-871. Fix ownership of Job/Task local files to have correct
+ group ownership according to the egid of the tasktracker.
+ (Vinod Kumar Vavilapalli via yhemanth)
diff --git a/src/c++/task-controller/task-controller.c b/src/c++/task-controller/task-controller.c
index cdf4f2d..09d8e06 100644
--- a/src/c++/task-controller/task-controller.c
+++ b/src/c++/task-controller/task-controller.c
@@ -333,7 +333,7 @@
continue;
}
- if (check_ownership(getuid(), getgid(), entry->fts_path) != 0) {
+ if (check_ownership(entry->fts_path) != 0) {
fprintf(LOGFILE,
"Invalid file path. %s not user/group owned by the tasktracker.\n",
entry->fts_path);
@@ -369,13 +369,13 @@
return INVALID_ARGUMENT_NUMBER;
}
+ gid_t tasktracker_gid = getegid(); // the group permissions of the binary.
+
if (get_user_details(user) < 0) {
fprintf(LOGFILE, "Couldn't get the user details of %s.\n", user);
return INVALID_USER_NAME;
}
- int tasktracker_gid = getgid();
-
char **local_dir = (char **) get_values(TT_SYS_DIR_KEY);
if (local_dir == NULL) {
@@ -485,7 +485,7 @@
}
}
- int tasktracker_gid = getgid();
+ gid_t tasktracker_gid = getegid(); // the group permissions of the binary.
if (secure_path(task_log_dir, user_detail->pw_uid, tasktracker_gid, S_IRWXU
| S_IRWXG, S_ISGID | S_IRWXU | S_IRWXG) != 0) {
// setgid on dirs but not files, 770. As of now, there are no files though
@@ -508,15 +508,18 @@
}
/*
- * Function to check if a user/group actually owns the file.
+ * Function to check if the TaskTracker actually owns the file.
*/
-int check_ownership(uid_t uid, gid_t gid, char *path) {
+int check_ownership(char *path) {
struct stat filestat;
if (stat(path, &filestat) != 0) {
return UNABLE_TO_STAT_FILE;
}
- // check user/group.
- if (uid != filestat.st_uid || gid != filestat.st_gid) {
+ // check user/group. User should be TaskTracker user, group can either be
+ // TaskTracker's primary group or the special group to which binary's
+ // permissions are set.
+ if (getuid() != filestat.st_uid || (getgid() != filestat.st_gid && getegid()
+ != filestat.st_gid)) {
return FILE_NOT_OWNED_BY_TASKTRACKER;
}
return 0;
@@ -540,7 +543,7 @@
return INVALID_USER_NAME;
}
- gid_t tasktracker_gid = getgid(); // TaskTracker's group-id
+ gid_t tasktracker_gid = getegid(); // the group permissions of the binary.
char **local_dir = (char **) get_values(TT_SYS_DIR_KEY);
if (local_dir == NULL) {
diff --git a/src/test/mapred/org/apache/hadoop/mapred/TestLocalizationWithLinuxTaskController.java b/src/test/mapred/org/apache/hadoop/mapred/TestLocalizationWithLinuxTaskController.java
index 9e57137..a3e7f83 100644
--- a/src/test/mapred/org/apache/hadoop/mapred/TestLocalizationWithLinuxTaskController.java
+++ b/src/test/mapred/org/apache/hadoop/mapred/TestLocalizationWithLinuxTaskController.java
@@ -30,7 +30,6 @@
import org.apache.hadoop.mapred.TaskController.JobInitializationContext;
import org.apache.hadoop.mapred.TaskController.TaskControllerContext;
import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.mapred.ClusterWithLinuxTaskController.MyLinuxTaskController;
import org.apache.hadoop.mapred.JvmManager.JvmEnv;
@@ -48,6 +47,8 @@
private File configFile;
private MyLinuxTaskController taskController;
+ private static String taskTrackerSpecialGroup;
+
@Override
protected void setUp()
throws Exception {
@@ -66,6 +67,7 @@
localDirs);
String execPath = path + "/task-controller";
taskController.setTaskControllerExe(execPath);
+ taskTrackerSpecialGroup = getFilePermissionAttrs(execPath)[2];
taskController.setConf(trackerFConf);
taskController.setup();
}
@@ -120,14 +122,12 @@
taskController.initializeJob(context);
// ///////////
- UserGroupInformation taskTrackerugi =
- UserGroupInformation.login(localizedJobConf);
for (String localDir : trackerFConf.getStrings("mapred.local.dir")) {
File jobDir =
new File(localDir, TaskTracker.getLocalJobDir(jobId.toString()));
// check the private permissions on the job directory
checkFilePermissions(jobDir.getAbsolutePath(), "dr-xrws---",
- localizedJobConf.getUser(), taskTrackerugi.getGroupNames()[0]);
+ localizedJobConf.getUser(), taskTrackerSpecialGroup);
}
// check the private permissions of various directories
@@ -139,7 +139,7 @@
dirs.add(new Path(jarsDir, "lib"));
for (Path dir : dirs) {
checkFilePermissions(dir.toUri().getPath(), "dr-xrws---",
- localizedJobConf.getUser(), taskTrackerugi.getGroupNames()[0]);
+ localizedJobConf.getUser(), taskTrackerSpecialGroup);
}
// job-work dir needs user writable permissions
@@ -147,7 +147,7 @@
lDirAlloc.getLocalPathToRead(TaskTracker.getJobWorkDir(jobId
.toString()), trackerFConf);
checkFilePermissions(jobWorkDir.toUri().getPath(), "drwxrws---",
- localizedJobConf.getUser(), taskTrackerugi.getGroupNames()[0]);
+ localizedJobConf.getUser(), taskTrackerSpecialGroup);
// check the private permissions of various files
List<Path> files = new ArrayList<Path>();
@@ -159,7 +159,7 @@
files.add(new Path(jarsDir, "lib" + Path.SEPARATOR + "lib2.jar"));
for (Path file : files) {
checkFilePermissions(file.toUri().getPath(), "-r-xrwx---",
- localizedJobConf.getUser(), taskTrackerugi.getGroupNames()[0]);
+ localizedJobConf.getUser(), taskTrackerSpecialGroup);
}
}
@@ -223,11 +223,9 @@
dirs.add(workDir);
dirs.add(new Path(workDir, "tmp"));
dirs.add(new Path(logFiles[1].getParentFile().getAbsolutePath()));
- UserGroupInformation taskTrackerugi =
- UserGroupInformation.login(localizedJobConf);
for (Path dir : dirs) {
checkFilePermissions(dir.toUri().getPath(), "drwxrws---",
- localizedJobConf.getUser(), taskTrackerugi.getGroupNames()[0]);
+ localizedJobConf.getUser(), taskTrackerSpecialGroup);
}
// check the private permissions of various files
@@ -237,7 +235,7 @@
.isTaskCleanupTask()), trackerFConf));
for (Path file : files) {
checkFilePermissions(file.toUri().getPath(), "-rwxrwx---",
- localizedJobConf.getUser(), taskTrackerugi.getGroupNames()[0]);
+ localizedJobConf.getUser(), taskTrackerSpecialGroup);
}
}
}
diff --git a/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java b/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java
index 36e4d50..da91194 100644
--- a/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java
+++ b/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java
@@ -131,7 +131,7 @@
FileUtil.fullyDelete(TEST_ROOT_DIR);
}
- private static String[] getFilePermissionAttrs(String path)
+ protected static String[] getFilePermissionAttrs(String path)
throws IOException {
String output = Shell.execCommand("stat", path, "-c", "%A:%U:%G");
return output.split(":|\n");
