mapreduce/src/test/mapred/org/apache/hadoop/mapred/QueueManagerTestUtils.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * <p/>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p/>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.mapred;

//import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Cluster;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.QueueState;
import org.apache.hadoop.mapreduce.SleepJob;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.security.UserGroupInformation;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.TransformerException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.dom.DOMSource;

import java.security.PrivilegedExceptionAction;
import java.util.Properties;
import java.util.Set;
import java.io.File;
import java.io.IOException;

//@Private
public class QueueManagerTestUtils {
  /**
   * Queue-configuration file for tests that start a cluster and wish to modify
   * the queue configuration. This file is always in the unit tests classpath,
   * so QueueManager started through JobTracker will automatically pick this up.
   */
  public static final String QUEUES_CONFIG_FILE_PATH = new File(System
      .getProperty("test.build.extraconf", "build/test/extraconf"),
      QueueManager.QUEUE_CONF_FILE_NAME).getAbsolutePath();

  private static final Log LOG = LogFactory.getLog(QueueManagerTestUtils.class);

  /**
   * Create and return a new instance of a DOM Document object to build a queue
   * tree with.
   * 
   * @return the created {@link Document}
   * @throws Exception
   */
  public static Document createDocument() throws Exception {
    Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder()
        .newDocument();
    return doc;
  }

  public static void createSimpleDocument(Document doc) throws Exception {
    Element queues = createQueuesNode(doc);

    // Create parent level queue q1.
    Element q1 = createQueue(doc, "q1");
    Properties props = new Properties();
    props.setProperty("capacity", "10");
    props.setProperty("maxCapacity", "35");
    q1.appendChild(createProperties(doc, props));
    queues.appendChild(q1);

    // Create another parent level p1
    Element p1 = createQueue(doc, "p1");

    // append child p11 to p1
    p1.appendChild(createQueue(doc, "p11"));

    Element p12 = createQueue(doc, "p12");

    p12.appendChild(createState(doc, QueueState.STOPPED.getStateName()));
    p12.appendChild(createAcls(doc,
        QueueConfigurationParser.ACL_SUBMIT_JOB_TAG, "u1"));
    p12.appendChild(createAcls(doc,
        QueueConfigurationParser.ACL_ADMINISTER_JOB_TAG, "u2"));

    // append p12 to p1.
    p1.appendChild(p12);

    queues.appendChild(p1);
  }

  static void createSimpleDocumentWithAcls(Document doc) {
    Element queues = createQueuesNode(doc);

    // Create parent level queue q1.
    Element q1 = createQueue(doc, "q1");
    Properties props = new Properties();
    props.setProperty("capacity", "10");
    props.setProperty("maxCapacity", "35");
    q1.appendChild(createProperties(doc, props));
    queues.appendChild(q1);

    // Create another parent level p1
    Element p1 = createQueue(doc, "p1");

    // append child p11 to p1
    Element p11 = createQueue(doc, "p11");
    p11.appendChild(createAcls(doc,
        QueueConfigurationParser.ACL_SUBMIT_JOB_TAG, "u1"));
    p11.appendChild(createAcls(doc,
        QueueConfigurationParser.ACL_ADMINISTER_JOB_TAG, "u2"));
    p1.appendChild(p11);

    // append child p12 to p1
    Element p12 = createQueue(doc, "p12");
    p12.appendChild(createState(doc, QueueState.RUNNING.getStateName()));
    p12.appendChild(createAcls(doc,
        QueueConfigurationParser.ACL_SUBMIT_JOB_TAG, "*"));
    p12.appendChild(createAcls(doc,
        QueueConfigurationParser.ACL_ADMINISTER_JOB_TAG, "*"));
    p1.appendChild(p12);

    // append child p13 to p1
    Element p13 = createQueue(doc, "p13");
    p13.appendChild(createState(doc, QueueState.RUNNING.getStateName()));
    p1.appendChild(p13);

    // append child p14 to p1
    Element p14 = createQueue(doc, "p14");
    p14.appendChild(createState(doc, QueueState.STOPPED.getStateName()));
    p1.appendChild(p14);

    queues.appendChild(p1);
  }

  /**
   * Creates all given queues as 1st level queues(no nesting)
   * @param doc         the queues config document
   * @param queueNames  the queues to be added to the queues config document
   * @param submitAcls  acl-submit-job acls for each of the queues
   * @param adminsAcls  acl-administer-jobs acls for each of the queues
   * @throws Exception
   */
  public static void createSimpleDocument(Document doc, String[] queueNames,
      String[] submitAcls, String[] adminsAcls) throws Exception {

    Element queues = createQueuesNode(doc);

    // Create all queues as 1st level queues(no nesting)
    for (int i = 0; i < queueNames.length; i++) {
      Element q = createQueue(doc, queueNames[i]);

      q.appendChild(createState(doc, QueueState.RUNNING.getStateName()));
      q.appendChild(createAcls(doc,
          QueueConfigurationParser.ACL_SUBMIT_JOB_TAG, submitAcls[i]));
      q.appendChild(createAcls(doc,
          QueueConfigurationParser.ACL_ADMINISTER_JOB_TAG, adminsAcls[i]));
      queues.appendChild(q);
    }
  }

  /**
   * Creates queues configuration file with given queues at 1st level(i.e.
   * no nesting of queues) and with the given queue acls.
   * @param queueNames        queue names which are to be configured
   * @param submitAclStrings  acl-submit-job acls for each of the queues
   * @param adminsAclStrings  acl-administer-jobs acls for each of the queues
   * @return Configuration    the queues configuration
   * @throws Exception
   */
  public static void createQueuesConfigFile(String[] queueNames,
      String[] submitAclStrings, String[] adminsAclStrings)
  throws Exception {
    if (queueNames.length > submitAclStrings.length ||
        queueNames.length > adminsAclStrings.length) {
      LOG.error("Number of queues is more than acls given.");
      return;
    }
    Document doc = createDocument();
    createSimpleDocument(doc, queueNames, submitAclStrings, adminsAclStrings);
    writeToFile(doc, QUEUES_CONFIG_FILE_PATH);
  }

  public static void refreshSimpleDocument(Document doc) throws Exception {
    Element queues = createQueuesNode(doc);

    // Create parent level queue q1.
    Element q1 = createQueue(doc, "q1");
    Properties props = new Properties();
    props.setProperty("capacity", "70");
    props.setProperty("maxCapacity", "35");
    q1.appendChild(createProperties(doc, props));
    queues.appendChild(q1);

    // Create another parent level p1
    Element p1 = createQueue(doc, "p1");

    // append child p11 to p1
    Element p11 = createQueue(doc, "p11");
    p11.appendChild(createState(doc, QueueState.STOPPED.getStateName()));
    p1.appendChild(p11);

    Element p12 = createQueue(doc, "p12");

    p12.appendChild(createState(doc, QueueState.RUNNING.getStateName()));
    p12.appendChild(createAcls(doc, "acl-submit-job", "u3"));
    p12.appendChild(createAcls(doc, "acl-administer-jobs", "u4"));

    // append p12 to p1.
    p1.appendChild(p12);

    queues.appendChild(p1);
  }

  /**
   * Create the root <queues></queues> element along with the
   * <aclsEnabled></aclsEnabled> element.
   * 
   * @param doc
   * @param enable
   * @return the created element.
   */
  public static Element createQueuesNode(Document doc) {
    Element queues = doc.createElement("queues");
    doc.appendChild(queues);
    return queues;
  }

  public static void writeToFile(Document doc, String filePath)
      throws TransformerException {
    Transformer trans = TransformerFactory.newInstance().newTransformer();
    trans.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
    trans.setOutputProperty(OutputKeys.INDENT, "yes");
    DOMSource source = new DOMSource(doc);
    trans.transform(source, new StreamResult(new File(filePath)));
  }

  public static Element createQueue(Document doc, String name) {
    Element queue = doc.createElement("queue");
    Element nameNode = doc.createElement("name");
    nameNode.setTextContent(name);
    queue.appendChild(nameNode);
    return queue;
  }

  public static Element createAcls(Document doc, String aclName,
      String listNames) {
    Element acls = doc.createElement(aclName);
    acls.setTextContent(listNames);
    return acls;
  }

  public static Element createState(Document doc, String state) {
    Element stateElement = doc.createElement("state");
    stateElement.setTextContent(state);
    return stateElement;
  }

  public static Element createProperties(Document doc, Properties props) {
    Element propsElement = doc.createElement("properties");
    if (props != null) {
      Set<String> propList = props.stringPropertyNames();
      for (String prop : propList) {
        Element property = doc.createElement("property");
        property.setAttribute("key", prop);
        property.setAttribute("value", (String) props.get(prop));
        propsElement.appendChild(property);
      }
    }
    return propsElement;
  }

  /**
   * Delete queues configuration file if exists
   */
  public static void deleteQueuesConfigFile() {
    if (new File(QUEUES_CONFIG_FILE_PATH).exists()) {
      new File(QUEUES_CONFIG_FILE_PATH).delete();
    }
  }

  /**
   * Write the given queueHierarchy to the given file.
   * 
   * @param filePath
   * 
   * @param rootQueues
   * @throws Exception
   */
  public static void writeQueueConfigurationFile(String filePath,
      JobQueueInfo[] rootQueues) throws Exception {
    Document doc = createDocument();
    Element queueElements = createQueuesNode(doc);
    for (JobQueueInfo rootQ : rootQueues) {
      queueElements.appendChild(QueueConfigurationParser.getQueueElement(doc,
          rootQ));
    }
    writeToFile(doc, filePath);
  }

  static Job submitSleepJob(final int numMappers, final int numReducers, final long mapSleepTime,
      final long reduceSleepTime, boolean shouldComplete, String userInfo,
      String queueName, Configuration clientConf) throws IOException,
      InterruptedException, ClassNotFoundException {
    clientConf.set(JTConfig.JT_IPC_ADDRESS, "localhost:"
        + miniMRCluster.getJobTrackerPort());
    UserGroupInformation ugi;
    if (userInfo != null) {
      String[] splits = userInfo.split(",");
      String[] groups = new String[splits.length - 1];
      System.arraycopy(splits, 1, groups, 0, splits.length - 1);
      ugi = UserGroupInformation.createUserForTesting(splits[0], groups);
    } else {
      ugi = UserGroupInformation.getCurrentUser();
    }
    if (queueName != null) {
      clientConf.set(JobContext.QUEUE_NAME, queueName);
    }
    final SleepJob sleep = new SleepJob();
    sleep.setConf(clientConf);
    
    Job job = ugi.doAs(new PrivilegedExceptionAction<Job>() {
        public Job run() throws IOException {
          return sleep.createJob(numMappers, numReducers, mapSleepTime,
              (int) mapSleepTime, reduceSleepTime, (int) reduceSleepTime);
      }});
    if (shouldComplete) {
      job.waitForCompletion(false);
    } else {
      job.submit();
      // miniMRCluster.getJobTrackerRunner().getJobTracker().jobsToComplete()[]
      Cluster cluster = new Cluster(miniMRCluster.createJobConf());
      JobStatus[] status = miniMRCluster.getJobTrackerRunner().getJobTracker()
          .jobsToComplete();
      JobID id = status[status.length -1].getJobID();
      Job newJob = cluster.getJob(id);
      cluster.close();
      return newJob;
    }
    return job;
  }

  static MiniMRCluster miniMRCluster;
}