tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandlerJobs.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.tez.auxservices;
import java.io.File;
import java.io.IOException;

import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
import org.apache.tez.client.TezClient;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.examples.OrderedWordCount;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import static org.apache.tez.test.TestTezJobs.generateOrderedWordCountInput;
import static org.apache.tez.test.TestTezJobs.verifyOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.test.MiniTezCluster;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

public class TestShuffleHandlerJobs {

  private static final Logger LOG = LoggerFactory.getLogger(TestShuffleHandlerJobs.class);

  protected static MiniTezCluster tezCluster;
  protected static MiniDFSCluster dfsCluster;

  private static Configuration conf = new Configuration();
  private static FileSystem remoteFs;
  private static int NUM_NMS = 5;
  private static int NUM_DNS = 5;
  @BeforeClass
  public static void setup() throws IOException {
    try {
      conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
      conf.setInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, 22);
      dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DNS)
          .format(true).build();
      remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
      throw new RuntimeException("problem starting mini dfs cluster", io);
    }

    if (!(new File(MiniTezCluster.APPJAR)).exists()) {
      LOG.info("MRAppJar " + MiniTezCluster.APPJAR
          + " not found. Not running test.");
      return;
    }

    if (tezCluster == null) {
      tezCluster = new MiniTezCluster(TestShuffleHandlerJobs.class.getName(), NUM_NMS,
          1, 1);
      Configuration conf = new Configuration();
      conf.set(YarnConfiguration.NM_AUX_SERVICES,
          ShuffleHandler.TEZ_SHUFFLE_SERVICEID);
      String serviceStr = String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
          ShuffleHandler.TEZ_SHUFFLE_SERVICEID);
      conf.set(serviceStr, ShuffleHandler.class.getName());
      conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
      conf.set("fs.defaultFS", remoteFs.getUri().toString());   // use HDFS
      conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l);
      tezCluster.init(conf);
      tezCluster.start();
    }

  }

  @AfterClass
  public static void tearDown() {
    if (tezCluster != null) {
      tezCluster.stop();
      tezCluster = null;
    }
    if (dfsCluster != null) {
      dfsCluster.shutdown();
      dfsCluster = null;
    }
  }
  @Test(timeout = 300000)
  public void testOrderedWordCount() throws Exception {
    String inputDirStr = "/tmp/owc-input/";
    Path inputDir = new Path(inputDirStr);
    Path stagingDirPath = new Path("/tmp/owc-staging-dir");
    remoteFs.mkdirs(inputDir);
    remoteFs.mkdirs(stagingDirPath);
    generateOrderedWordCountInput(inputDir, remoteFs);

    String outputDirStr = "/tmp/owc-output/";
    Path outputDir = new Path(outputDirStr);

    TezConfiguration tezConf = new TezConfiguration(tezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.set(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, ShuffleHandler.TEZ_SHUFFLE_SERVICEID);
    tezConf.setBoolean(TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION, true);
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
    tezConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, false);
    tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
    TezClient tezSession = TezClient.create("WordCountTest", tezConf);
    tezSession.start();
    try {
      final OrderedWordCount job = new OrderedWordCount();
      Assert.assertTrue("OrderedWordCount failed", job.run(tezConf, new String[]{"-counter",
              inputDirStr, outputDirStr, "10"}, tezSession)==0);
      verifyOutput(outputDir, remoteFs);
      tezSession.stop();
      ClientRMService rmService = tezCluster.getResourceManager().getClientRMService();
      boolean isAppComplete = false;
      while(!isAppComplete) {
        GetApplicationReportResponse resp = rmService.getApplicationReport(
            new GetApplicationReportRequest() {
              @Override
              public ApplicationId getApplicationId() {
                return job.getAppId();
              }

              @Override
              public void setApplicationId(ApplicationId applicationId) {
              }
            });
        if (resp.getApplicationReport().getYarnApplicationState() == YarnApplicationState.FINISHED) {
          isAppComplete = true;
        }
        Thread.sleep(100);
      }
      for(int i = 0; i < NUM_NMS; i++) {
        String appPath = tezCluster.getTestWorkDir() + "/" + this.getClass().getName()
            + "-localDir-nm-" + i + "_0/usercache/" + UserGroupInformation.getCurrentUser().getUserName()
            + "/appcache/" + job.getAppId();
        String dagPathStr = appPath + "/dag_1";

        File fs = new File(dagPathStr);
        Assert.assertFalse(fs.exists());
        fs = new File(appPath);
        Assert.assertTrue(fs.exists());
      }
    } finally {
      remoteFs.delete(stagingDirPath, true);
    }
  }
}