| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.mapred; |
| |
| import java.io.IOException; |
| import java.io.UnsupportedEncodingException; |
| import java.net.URLEncoder; |
| import java.security.PrivilegedExceptionAction; |
| import java.util.Collection; |
| import java.util.Date; |
| import java.util.Iterator; |
| import java.util.LinkedHashMap; |
| import java.util.Map; |
| |
| import javax.servlet.RequestDispatcher; |
| import javax.servlet.ServletException; |
| import javax.servlet.http.HttpServletRequest; |
| import javax.servlet.http.HttpServletResponse; |
| import javax.servlet.jsp.JspWriter; |
| |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| import org.apache.hadoop.fs.FileSystem; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.http.HtmlQuoting; |
| import org.apache.hadoop.mapred.JobHistory.JobInfo; |
| import org.apache.hadoop.mapred.JobHistory.Keys; |
| import org.apache.hadoop.mapred.JobTracker.RetireJobInfo; |
| import org.apache.hadoop.mapreduce.JobACL; |
| import org.apache.hadoop.security.AccessControlException; |
| import org.apache.hadoop.security.UserGroupInformation; |
| import org.apache.hadoop.security.authorize.AccessControlList; |
| import org.apache.hadoop.util.ServletUtil; |
| import org.apache.hadoop.util.StringUtils; |
| |
| class JSPUtil { |
| static final String PRIVATE_ACTIONS_KEY = "webinterface.private.actions"; |
| |
| //LRU based cache |
| private static final Map<String, JobInfo> jobHistoryCache = |
| new LinkedHashMap<String, JobInfo>(); |
| |
| private static final Log LOG = LogFactory.getLog(JSPUtil.class); |
| |
| /** |
| * Wraps the {@link JobInProgress} object and contains boolean for |
| * 'job view access' allowed or not. |
| * This class is only for usage by JSPs and Servlets. |
| */ |
| static class JobWithViewAccessCheck { |
| private JobInProgress job = null; |
| |
| // true if user is authorized to view this job |
| private boolean isViewAllowed = true; |
| |
| JobWithViewAccessCheck(JobInProgress job) { |
| this.job = job; |
| } |
| |
| JobInProgress getJob() { |
| return job; |
| } |
| |
| boolean isViewJobAllowed() { |
| return isViewAllowed; |
| } |
| |
| void setViewAccess(boolean isViewAllowed) { |
| this.isViewAllowed = isViewAllowed; |
| } |
| } |
| |
| /** |
| * Validates if current user can view the job. |
| * If user is not authorized to view the job, this method will modify the |
| * response and forwards to an error page and returns Job with |
| * viewJobAccess flag set to false. |
| * @return JobWithViewAccessCheck object(contains JobInProgress object and |
| * viewJobAccess flag). Callers of this method will check the flag |
| * and decide if view should be allowed or not. Job will be null if |
| * the job with given jobid doesnot exist at the JobTracker. |
| */ |
| public static JobWithViewAccessCheck checkAccessAndGetJob(final JobTracker jt, |
| JobID jobid, HttpServletRequest request, HttpServletResponse response) |
| throws ServletException, IOException { |
| final JobInProgress job = jt.getJob(jobid); |
| JobWithViewAccessCheck myJob = new JobWithViewAccessCheck(job); |
| |
| String user = request.getRemoteUser(); |
| if (user != null && job != null && jt.areACLsEnabled()) { |
| final UserGroupInformation ugi = |
| UserGroupInformation.createRemoteUser(user); |
| try { |
| ugi.doAs(new PrivilegedExceptionAction<Void>() { |
| public Void run() throws IOException, ServletException { |
| |
| // checks job view permission |
| jt.getACLsManager().checkAccess(job, ugi, |
| Operation.VIEW_JOB_DETAILS); |
| return null; |
| } |
| }); |
| } catch (AccessControlException e) { |
| String errMsg = "User " + ugi.getShortUserName() + |
| " failed to view " + jobid + "!<br><br>" + e.getMessage() + |
| "<hr><a href=\"jobtracker.jsp\">Go back to JobTracker</a><br>"; |
| JSPUtil.setErrorAndForward(errMsg, request, response); |
| myJob.setViewAccess(false); |
| } catch (InterruptedException e) { |
| String errMsg = " Interrupted while trying to access " + jobid + |
| "<hr><a href=\"jobtracker.jsp\">Go back to JobTracker</a><br>"; |
| JSPUtil.setErrorAndForward(errMsg, request, response); |
| myJob.setViewAccess(false); |
| } |
| } |
| return myJob; |
| } |
| |
| /** |
| * Sets error code SC_UNAUTHORIZED in response and forwards to |
| * error page which contains error message and a back link. |
| */ |
| public static void setErrorAndForward(String errMsg, |
| HttpServletRequest request, HttpServletResponse response) |
| throws ServletException, IOException { |
| request.setAttribute("error.msg", errMsg); |
| RequestDispatcher dispatcher = request.getRequestDispatcher( |
| "/job_authorization_error.jsp"); |
| response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); |
| dispatcher.forward(request, response); |
| } |
| |
| /** |
| * Method used to process the request from the job page based on the |
| * request which it has received. For example like changing priority. |
| * |
| * @param request HTTP request Object. |
| * @param response HTTP response object. |
| * @param tracker {@link JobTracker} instance |
| * @throws IOException |
| * @throws InterruptedException |
| * @throws ServletException |
| */ |
| public static void processButtons(HttpServletRequest request, |
| HttpServletResponse response, final JobTracker tracker) |
| throws IOException, InterruptedException, ServletException { |
| |
| String user = request.getRemoteUser(); |
| if (privateActionsAllowed(tracker.conf) |
| && request.getParameter("killJobs") != null) { |
| String[] jobs = request.getParameterValues("jobCheckBox"); |
| if (jobs != null) { |
| boolean notAuthorized = false; |
| String errMsg = "User " + user |
| + " failed to kill the following job(s)!<br><br>"; |
| for (String job : jobs) { |
| final JobID jobId = JobID.forName(job); |
| if (user != null) { |
| UserGroupInformation ugi = |
| UserGroupInformation.createRemoteUser(user); |
| try { |
| ugi.doAs(new PrivilegedExceptionAction<Void>() { |
| public Void run() throws IOException{ |
| |
| tracker.killJob(jobId);// checks job modify permission |
| return null; |
| } |
| }); |
| } catch(AccessControlException e) { |
| errMsg = errMsg.concat("<br>" + e.getMessage()); |
| notAuthorized = true; |
| // We don't return right away so that we can try killing other |
| // jobs that are requested to be killed. |
| continue; |
| } |
| } |
| else {// no authorization needed |
| tracker.killJob(jobId); |
| } |
| } |
| if (notAuthorized) {// user is not authorized to kill some/all of jobs |
| errMsg = errMsg.concat( |
| "<br><hr><a href=\"jobtracker.jsp\">Go back to JobTracker</a><br>"); |
| setErrorAndForward(errMsg, request, response); |
| return; |
| } |
| } |
| } |
| |
| if (privateActionsAllowed(tracker.conf) && |
| request.getParameter("changeJobPriority") != null) { |
| String[] jobs = request.getParameterValues("jobCheckBox"); |
| if (jobs != null) { |
| final JobPriority jobPri = JobPriority.valueOf(request |
| .getParameter("setJobPriority")); |
| boolean notAuthorized = false; |
| String errMsg = "User " + user |
| + " failed to set priority for the following job(s)!<br><br>"; |
| |
| for (String job : jobs) { |
| final JobID jobId = JobID.forName(job); |
| if (user != null) { |
| UserGroupInformation ugi = UserGroupInformation. |
| createRemoteUser(user); |
| try { |
| ugi.doAs(new PrivilegedExceptionAction<Void>() { |
| public Void run() throws IOException{ |
| |
| // checks job modify permission |
| tracker.setJobPriority(jobId, jobPri); |
| return null; |
| } |
| }); |
| } catch(AccessControlException e) { |
| errMsg = errMsg.concat("<br>" + e.getMessage()); |
| notAuthorized = true; |
| // We don't return right away so that we can try operating on |
| // other jobs. |
| continue; |
| } |
| } |
| else {// no authorization needed |
| tracker.setJobPriority(jobId, jobPri); |
| } |
| } |
| if (notAuthorized) {// user is not authorized to kill some/all of jobs |
| errMsg = errMsg.concat( |
| "<br><hr><a href=\"jobtracker.jsp\">Go back to JobTracker</a><br>"); |
| setErrorAndForward(errMsg, request, response); |
| return; |
| } |
| } |
| } |
| } |
| |
| /** |
| * Method used to generate the Job table for Job pages. |
| * |
| * @param label display heading to be used in the job table. |
| * @param jobs vector of jobs to be displayed in table. |
| * @param refresh refresh interval to be used in jobdetails page. |
| * @param rowId beginning row id to be used in the table. |
| * @return |
| * @throws IOException |
| */ |
| public static String generateJobTable(String label, Collection<JobInProgress> jobs |
| , int refresh, int rowId, JobConf conf) throws IOException { |
| |
| boolean isModifiable = label.equals("Running") |
| && privateActionsAllowed(conf); |
| StringBuffer sb = new StringBuffer(); |
| sb.append("<table border=\"1\" cellpadding=\"5\" cellspacing=\"0\" class=\"sortable\">\n"); |
| |
| if (jobs.size() > 0) { |
| if (isModifiable) { |
| sb.append("<form action=\"/jobtracker.jsp\" onsubmit=\"return confirmAction();\" method=\"POST\">"); |
| sb.append("<tr>"); |
| sb.append("<td><input type=\"Button\" onclick=\"selectAll()\" " + |
| "value=\"Select All\" id=\"checkEm\"></td>"); |
| sb.append("<td>"); |
| sb.append("<input type=\"submit\" name=\"killJobs\" value=\"Kill Selected Jobs\">"); |
| sb.append("</td"); |
| sb.append("<td><nobr>"); |
| sb.append("<select name=\"setJobPriority\">"); |
| |
| for (JobPriority prio : JobPriority.values()) { |
| sb.append("<option" |
| + (JobPriority.NORMAL == prio ? " selected=\"selected\">" : ">") |
| + prio + "</option>"); |
| } |
| |
| sb.append("</select>"); |
| sb.append("<input type=\"submit\" name=\"changeJobPriority\" " + |
| "value=\"Change\">"); |
| sb.append("</nobr></td>"); |
| sb.append("<td colspan=\"10\"> </td>"); |
| sb.append("</tr>"); |
| sb.append("<td> </td>"); |
| } else { |
| sb.append("<tr>"); |
| } |
| |
| sb.append("<td><b>Jobid</b></td><td><b>Priority" + |
| "</b></td><td><b>User</b></td>"); |
| sb.append("<td><b>Name</b></td>"); |
| sb.append("<td><b>Map % Complete</b></td>"); |
| sb.append("<td><b>Map Total</b></td>"); |
| sb.append("<td><b>Maps Completed</b></td>"); |
| sb.append("<td><b>Reduce % Complete</b></td>"); |
| sb.append("<td><b>Reduce Total</b></td>"); |
| sb.append("<td><b>Reduces Completed</b></td>"); |
| sb.append("<td><b>Job Scheduling Information</b></td>"); |
| sb.append("<td><b>Diagnostic Info </b></td>"); |
| sb.append("</tr>\n"); |
| |
| for (Iterator<JobInProgress> it = jobs.iterator(); it.hasNext(); ++rowId) { |
| JobInProgress job = it.next(); |
| JobProfile profile = job.getProfile(); |
| JobStatus status = job.getStatus(); |
| JobID jobid = profile.getJobID(); |
| |
| int desiredMaps = job.desiredMaps(); |
| int desiredReduces = job.desiredReduces(); |
| int completedMaps = job.finishedMaps(); |
| int completedReduces = job.finishedReduces(); |
| String name = HtmlQuoting.quoteHtmlChars(profile.getJobName()); |
| String jobpri = job.getPriority().toString(); |
| String schedulingInfo = |
| HtmlQuoting.quoteHtmlChars(job.getStatus().getSchedulingInfo()); |
| String diagnosticInfo = |
| HtmlQuoting.quoteHtmlChars(job.getStatus().getFailureInfo()); |
| if (isModifiable) { |
| sb.append("<tr><td><input TYPE=\"checkbox\" " + |
| "onclick=\"checkButtonVerbage()\" " + |
| "name=\"jobCheckBox\" value=" |
| + jobid + "></td>"); |
| } else { |
| sb.append("<tr>"); |
| } |
| |
| sb.append("<td id=\"job_" + rowId |
| + "\"><a href=\"jobdetails.jsp?jobid=" + jobid + "&refresh=" |
| + refresh + "\">" + jobid + "</a></td>" + "<td id=\"priority_" |
| + rowId + "\">" + jobpri + "</td>" + "<td id=\"user_" + rowId |
| + "\">" + HtmlQuoting.quoteHtmlChars(profile.getUser()) + |
| "</td>" + "<td id=\"name_" + rowId |
| + "\">" + ("".equals(name) ? " " : name) + "</td>" + "<td>" |
| + StringUtils.formatPercent(status.mapProgress(), 2) |
| + ServletUtil.percentageGraph(status.mapProgress() * 100, 80) |
| + "</td><td>" + desiredMaps + "</td><td>" + completedMaps |
| + "</td><td>" |
| + StringUtils.formatPercent(status.reduceProgress(), 2) |
| + ServletUtil.percentageGraph(status.reduceProgress() * 100, 80) |
| + "</td><td>" + desiredReduces + "</td><td> " + completedReduces |
| + "</td><td>" + schedulingInfo |
| + "</td><td>" + diagnosticInfo + "</td></tr>\n"); |
| } |
| if (isModifiable) { |
| sb.append("</form>\n"); |
| } |
| } else { |
| sb.append("<tr><td align=\"center\" colspan=\"8\"><i>none</i>" + |
| "</td></tr>\n"); |
| } |
| sb.append("</table>\n"); |
| |
| return sb.toString(); |
| } |
| |
| @SuppressWarnings("unchecked") |
| public static String generateRetiredJobTable(JobTracker tracker, int rowId) |
| throws IOException { |
| |
| StringBuffer sb = new StringBuffer(); |
| sb.append("<table border=\"1\" cellpadding=\"5\" cellspacing=\"0\">\n"); |
| |
| Iterator<RetireJobInfo> iterator = |
| tracker.retireJobs.getAll().descendingIterator(); |
| if (!iterator.hasNext()) { |
| sb.append("<tr><td align=\"center\" colspan=\"8\"><i>none</i>" + |
| "</td></tr>\n"); |
| } else { |
| sb.append("<tr>"); |
| |
| sb.append("<td><b>Jobid</b></td>"); |
| sb.append("<td><b>Priority</b></td>"); |
| sb.append("<td><b>User</b></td>"); |
| sb.append("<td><b>Name</b></td>"); |
| sb.append("<td><b>State</b></td>"); |
| sb.append("<td><b>Start Time</b></td>"); |
| sb.append("<td><b>Finish Time</b></td>"); |
| sb.append("<td><b>Map % Complete</b></td>"); |
| sb.append("<td><b>Reduce % Complete</b></td>"); |
| sb.append("<td><b>Job Scheduling Information</b></td>"); |
| sb.append("<td><b>Diagnostic Info </b></td>"); |
| sb.append("</tr>\n"); |
| for (int i = 0; i < 100 && iterator.hasNext(); i++) { |
| RetireJobInfo info = iterator.next(); |
| String historyFile = info.getHistoryFile(); |
| String historyFileUrl = null; |
| if (historyFile != null && !historyFile.equals("")) { |
| try { |
| historyFileUrl = URLEncoder.encode(info.getHistoryFile(), "UTF-8"); |
| } catch (UnsupportedEncodingException e) { |
| LOG.warn("Can't create history url ", e); |
| } |
| } |
| sb.append("<tr>"); |
| sb.append( |
| "<td id=\"job_" + rowId + "\">" + |
| |
| (historyFileUrl == null ? "" : |
| "<a href=\"" + JobHistoryServer.getHistoryUrlPrefix(tracker.conf) + |
| "/jobdetailshistory.jsp?logFile=" + historyFileUrl + "\">") + |
| |
| info.status.getJobId() + "</a></td>" + |
| |
| "<td id=\"priority_" + rowId + "\">" + |
| info.status.getJobPriority().toString() + "</td>" + |
| "<td id=\"user_" + rowId + "\">" + |
| HtmlQuoting.quoteHtmlChars(info.profile.getUser()) + "</td>" + |
| "<td id=\"name_" + rowId + "\">" + |
| HtmlQuoting.quoteHtmlChars(info.profile.getJobName()) + "</td>" + |
| "<td>" + JobStatus.getJobRunState(info.status.getRunState()) |
| + "</td>" + |
| "<td>" + new Date(info.status.getStartTime()) + "</td>" + |
| "<td>" + new Date(info.finishTime) + "</td>" + |
| |
| "<td>" + StringUtils.formatPercent(info.status.mapProgress(), 2) |
| + ServletUtil.percentageGraph(info.status.mapProgress() * 100, 80) + |
| "</td>" + |
| |
| "<td>" + StringUtils.formatPercent(info.status.reduceProgress(), 2) |
| + ServletUtil.percentageGraph( |
| info.status.reduceProgress() * 100, 80) + |
| "</td>" + |
| |
| "<td>" + |
| HtmlQuoting.quoteHtmlChars(info.status.getSchedulingInfo()) + |
| "</td>" + |
| "<td>" + HtmlQuoting.quoteHtmlChars(info.status.getFailureInfo()) + |
| "</td></tr>\n"); |
| rowId++; |
| } |
| } |
| sb.append("</table>\n"); |
| return sb.toString(); |
| } |
| |
| static Path getJobConfFilePath(Path logFile) { |
| return JobHistory.confPathFromLogFilePath(logFile); |
| } |
| |
| /** |
| * Read a job-history log file and construct the corresponding {@link JobInfo} |
| * . Also cache the {@link JobInfo} for quick serving further requests. |
| * |
| * @param logFile |
| * @param fs |
| * @return JobInfo |
| * @throws IOException |
| */ |
| static JobInfo getJobInfo(Path logFile, FileSystem fs, |
| JobConf jobConf, ACLsManager acLsManager, String user) throws IOException { |
| String jobid = getJobID(logFile.getName()); |
| JobInfo jobInfo = null; |
| synchronized(jobHistoryCache) { |
| jobInfo = jobHistoryCache.remove(jobid); |
| if (jobInfo == null) { |
| jobInfo = new JobHistory.JobInfo(jobid); |
| LOG.info("Loading Job History file "+jobid + ". Cache size is " + |
| jobHistoryCache.size()); |
| DefaultJobHistoryParser.parseJobTasks(logFile.toUri().getPath(), |
| jobInfo, fs); |
| } |
| jobHistoryCache.put(jobid, jobInfo); |
| int CACHE_SIZE = |
| jobConf.getInt("mapred.job.tracker.jobhistory.lru.cache.size", 5); |
| if (jobHistoryCache.size() > CACHE_SIZE) { |
| Iterator<Map.Entry<String, JobInfo>> it = |
| jobHistoryCache.entrySet().iterator(); |
| String removeJobId = it.next().getKey(); |
| it.remove(); |
| LOG.info("Job History file removed form cache "+removeJobId); |
| } |
| } |
| |
| UserGroupInformation currentUser; |
| if (user == null) { |
| currentUser = UserGroupInformation.getCurrentUser(); |
| } else { |
| currentUser = UserGroupInformation.createRemoteUser(user); |
| } |
| |
| // Authorize the user for view access of this job |
| acLsManager.checkAccess(jobid, currentUser, |
| jobInfo.getJobQueue(), Operation.VIEW_JOB_DETAILS, |
| jobInfo.get(Keys.USER), jobInfo.getJobACLs().get(JobACL.VIEW_JOB)); |
| |
| return jobInfo; |
| } |
| |
| /** |
| * Check the access for users to view job-history pages. |
| * |
| * @param request |
| * @param response |
| * @param fs |
| * @param logFile |
| * @return the job if authorization is disabled or if the authorization checks |
| * pass. Otherwise return null. |
| * @throws IOException |
| * @throws InterruptedException |
| * @throws ServletException |
| */ |
| static JobInfo checkAccessAndGetJobInfo(HttpServletRequest request, |
| HttpServletResponse response, final JobConf jobConf, |
| final ACLsManager acLsManager, final FileSystem fs, |
| final Path logFile) throws IOException, |
| InterruptedException, ServletException { |
| String jobid = getJobID(logFile.getName()); |
| String user = request.getRemoteUser(); |
| JobInfo job = null; |
| if (user != null) { |
| try { |
| job = JSPUtil.getJobInfo(logFile, fs, jobConf, acLsManager, user); |
| } catch (AccessControlException e) { |
| String trackerAddress = jobConf.get("mapred.job.tracker.http.address"); |
| String errMsg = |
| String.format( |
| "User %s failed to view %s!<br><br>%s" |
| + "<hr>" |
| + "<a href=\"jobhistory.jsp\">Go back to JobHistory</a><br>" |
| + "<a href=\"http://" + trackerAddress + |
| "/jobtracker.jsp\">Go back to JobTracker</a>", |
| user, jobid, e.getMessage()); |
| JSPUtil.setErrorAndForward(errMsg, request, response); |
| return null; |
| } |
| } else { |
| // no authorization needed |
| job = JSPUtil.getJobInfo(logFile, fs, jobConf, acLsManager, null); |
| } |
| return job; |
| } |
| |
| static String getJobID(String historyFileName) { |
| return JobHistory.jobIdNameFromLogFileName(historyFileName); |
| } |
| |
| static String getUserName(String historyFileName) { |
| return JobHistory.userNameFromLogFileName(historyFileName); |
| } |
| |
| static String getJobName(String historyFileName) { |
| return JobHistory.jobNameFromLogFileName(historyFileName); |
| } |
| |
| /** |
| * Nicely print the Job-ACLs |
| * @param tracker |
| * @param jobAcls |
| * @param out |
| * @throws IOException |
| */ |
| static void printJobACLs(JobTracker tracker, |
| Map<JobACL, AccessControlList> jobAcls, JspWriter out) |
| throws IOException { |
| if (tracker.areACLsEnabled()) { |
| printJobACLsInternal(jobAcls, out); |
| } |
| else { |
| out.print("<b>Job-ACLs: " + new AccessControlList("*").toString() |
| + "</b><br>"); |
| } |
| } |
| |
| static void printJobACLs(JobConf conf, |
| Map<JobACL, AccessControlList> jobAcls, JspWriter out) |
| throws IOException { |
| if (conf.getBoolean(JobConf.MR_ACLS_ENABLED, false)) { |
| printJobACLsInternal(jobAcls, out); |
| } |
| else { |
| out.print("<b>Job-ACLs: " + new AccessControlList("*").toString() |
| + "</b><br>"); |
| } |
| } |
| |
| private static void printJobACLsInternal(Map<JobACL, AccessControlList> jobAcls, |
| JspWriter out) throws IOException { |
| // Display job-view-acls and job-modify-acls configured for this job |
| out.print("<b>Job-ACLs:</b><br>"); |
| for (JobACL aclName : JobACL.values()) { |
| String aclConfigName = aclName.getAclName(); |
| AccessControlList aclConfigured = jobAcls.get(aclName); |
| if (aclConfigured != null) { |
| String aclStr = aclConfigured.toString(); |
| out.print(" " + aclConfigName + ": " |
| + aclStr + "<br>"); |
| } |
| } |
| } |
| |
| static boolean privateActionsAllowed(JobConf conf) { |
| return conf.getBoolean(PRIVATE_ACTIONS_KEY, false); |
| } |
| } |
