MAPREDUCE-4738. Part1. Uncomment disabled unit tests. (sseth)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-3902@1400227 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/MRAppBenchmark.java
index 9010820..e4d96a2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/MRAppBenchmark.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/MRAppBenchmark.java
@@ -1,283 +1,283 @@
-///**
-//* Licensed to the Apache Software Foundation (ASF) under one
-//* or more contributor license agreements. See the NOTICE file
-//* distributed with this work for additional information
-//* regarding copyright ownership. The ASF licenses this file
-//* to you under the Apache License, Version 2.0 (the
-//* "License"); you may not use this file except in compliance
-//* with the License. You may obtain a copy of the License at
-//*
-//* http://www.apache.org/licenses/LICENSE-2.0
-//*
-//* Unless required by applicable law or agreed to in writing, software
-//* distributed under the License is distributed on an "AS IS" BASIS,
-//* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//* See the License for the specific language governing permissions and
-//* limitations under the License.
-//*/
-//
-//package org.apache.hadoop.mapreduce.v2.app2;
-//
-//import java.util.ArrayList;
-//import java.util.List;
-//import java.util.concurrent.BlockingQueue;
-//import java.util.concurrent.LinkedBlockingQueue;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.mapreduce.v2.api.records.JobState;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-//import org.apache.hadoop.mapreduce.v2.app2.client.ClientService;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Job;
-//import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptContainerAssignedEvent;
-//import org.apache.hadoop.mapreduce.v2.app2.rm.ContainerAllocator;
-//import org.apache.hadoop.mapreduce.v2.app2.rm.ContainerAllocatorEvent;
-//import org.apache.hadoop.mapreduce.v2.app2.rm.RMContainerAllocator;
-//import org.apache.hadoop.yarn.YarnException;
-//import org.apache.hadoop.yarn.api.AMRMProtocol;
-//import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-//import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-//import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
-//import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
-//import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
-//import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-//import org.apache.hadoop.yarn.api.records.AMResponse;
-//import org.apache.hadoop.yarn.api.records.Container;
-//import org.apache.hadoop.yarn.api.records.ContainerId;
-//import org.apache.hadoop.yarn.api.records.NodeId;
-//import org.apache.hadoop.yarn.api.records.ResourceRequest;
-//import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-//import org.apache.hadoop.yarn.factories.RecordFactory;
-//import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-//import org.apache.hadoop.yarn.service.AbstractService;
-//import org.apache.hadoop.yarn.util.BuilderUtils;
-//import org.apache.hadoop.yarn.util.Records;
-//import org.apache.log4j.Level;
-//import org.apache.log4j.LogManager;
-//import org.apache.log4j.Logger;
-//import org.junit.Test;
-//
-//public class MRAppBenchmark {
-//
-// private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-//
-// /**
-// * Runs memory and time benchmark with Mock MRApp.
-// */
-// public void run(MRApp app) throws Exception {
-// Logger rootLogger = LogManager.getRootLogger();
-// rootLogger.setLevel(Level.WARN);
-// long startTime = System.currentTimeMillis();
-// Job job = app.submit(new Configuration());
-// while (!job.getReport().getJobState().equals(JobState.SUCCEEDED)) {
-// printStat(job, startTime);
-// Thread.sleep(2000);
-// }
-// printStat(job, startTime);
-// }
-//
-// private void printStat(Job job, long startTime) throws Exception {
-// long currentTime = System.currentTimeMillis();
-// Runtime.getRuntime().gc();
-// long mem = Runtime.getRuntime().totalMemory()
-// - Runtime.getRuntime().freeMemory();
-// System.out.println("JobState:" + job.getState() +
-// " CompletedMaps:" + job.getCompletedMaps() +
-// " CompletedReduces:" + job.getCompletedReduces() +
-// " Memory(total-free)(KB):" + mem/1024 +
-// " ElapsedTime(ms):" + (currentTime - startTime));
-// }
-//
-// //Throttles the maximum number of concurrent running tasks.
-// //This affects the memory requirement since
-// //org.apache.hadoop.mapred.MapTask/ReduceTask is loaded in memory for all
-// //running task and discarded once the task is launched.
-// static class ThrottledMRApp extends MRApp {
-//
-// int maxConcurrentRunningTasks;
-// volatile int concurrentRunningTasks;
-// ThrottledMRApp(int maps, int reduces, int maxConcurrentRunningTasks) {
-// super(maps, reduces, true, "ThrottledMRApp", true);
-// this.maxConcurrentRunningTasks = maxConcurrentRunningTasks;
-// }
-//
-// @Override
-// protected void attemptLaunched(TaskAttemptId attemptID) {
-// super.attemptLaunched(attemptID);
-// //the task is launched and sends done immediately
-// concurrentRunningTasks--;
-// }
-//
-// @Override
-// protected ContainerAllocator createContainerAllocator(
-// ClientService clientService, AppContext context) {
-// return new ThrottledContainerAllocator();
-// }
-//
-// class ThrottledContainerAllocator extends AbstractService
-// implements ContainerAllocator {
-// private int containerCount;
-// private Thread thread;
-// private BlockingQueue<ContainerAllocatorEvent> eventQueue =
-// new LinkedBlockingQueue<ContainerAllocatorEvent>();
-// public ThrottledContainerAllocator() {
-// super("ThrottledContainerAllocator");
-// }
-// @Override
-// public void handle(ContainerAllocatorEvent event) {
-// try {
-// eventQueue.put(event);
-// } catch (InterruptedException e) {
-// throw new YarnException(e);
-// }
-// }
-// @Override
-// public void start() {
-// thread = new Thread(new Runnable() {
-// @Override
-// public void run() {
-// ContainerAllocatorEvent event = null;
-// while (!Thread.currentThread().isInterrupted()) {
-// try {
-// if (concurrentRunningTasks < maxConcurrentRunningTasks) {
-// event = eventQueue.take();
-// ContainerId cId =
-// recordFactory.newRecordInstance(ContainerId.class);
-// cId.setApplicationAttemptId(
-// getContext().getApplicationAttemptId());
-// cId.setId(containerCount++);
-// //System.out.println("Allocating " + containerCount);
-//
-// Container container =
-// recordFactory.newRecordInstance(Container.class);
-// container.setId(cId);
-// NodeId nodeId = recordFactory.newRecordInstance(NodeId.class);
-// nodeId.setHost("dummy");
-// nodeId.setPort(1234);
-// container.setNodeId(nodeId);
-// container.setContainerToken(null);
-// container.setNodeHttpAddress("localhost:8042");
-// getContext().getEventHandler()
-// .handle(
-// new TaskAttemptContainerAssignedEvent(event
-// .getAttemptID(), container, null));
-// concurrentRunningTasks++;
-// } else {
-// Thread.sleep(1000);
-// }
-// } catch (InterruptedException e) {
-// System.out.println("Returning, interrupted");
-// return;
-// }
-// }
-// }
-// });
-// thread.start();
-// super.start();
-// }
-//
-// @Override
-// public void stop() {
-// thread.interrupt();
-// super.stop();
-// }
-// }
-// }
-//
-// @Test
-// public void benchmark1() throws Exception {
-// int maps = 100; // Adjust for benchmarking. Start with thousands.
-// int reduces = 0;
-// System.out.println("Running benchmark with maps:"+maps +
-// " reduces:"+reduces);
-// run(new MRApp(maps, reduces, true, this.getClass().getName(), true) {
-//
-// @Override
-// protected ContainerAllocator createContainerAllocator(
-// ClientService clientService, AppContext context) {
-// return new RMContainerAllocator(clientService, context) {
-// @Override
-// protected AMRMProtocol createSchedulerProxy() {
-// return new AMRMProtocol() {
-//
-// @Override
-// public RegisterApplicationMasterResponse
-// registerApplicationMaster(
-// RegisterApplicationMasterRequest request)
-// throws YarnRemoteException {
-// RegisterApplicationMasterResponse response =
-// Records.newRecord(RegisterApplicationMasterResponse.class);
-// response.setMinimumResourceCapability(BuilderUtils
-// .newResource(1024));
-// response.setMaximumResourceCapability(BuilderUtils
-// .newResource(10240));
-// return response;
-// }
-//
-// @Override
-// public FinishApplicationMasterResponse finishApplicationMaster(
-// FinishApplicationMasterRequest request)
-// throws YarnRemoteException {
-// FinishApplicationMasterResponse response =
-// Records.newRecord(FinishApplicationMasterResponse.class);
-// return response;
-// }
-//
-// @Override
-// public AllocateResponse allocate(AllocateRequest request)
-// throws YarnRemoteException {
-//
-// AllocateResponse response =
-// Records.newRecord(AllocateResponse.class);
-// List<ResourceRequest> askList = request.getAskList();
-// List<Container> containers = new ArrayList<Container>();
-// for (ResourceRequest req : askList) {
-// if (req.getHostName() != "*") {
-// continue;
-// }
-// int numContainers = req.getNumContainers();
-// for (int i = 0; i < numContainers; i++) {
-// ContainerId containerId =
-// BuilderUtils.newContainerId(
-// request.getApplicationAttemptId(),
-// request.getResponseId() + i);
-// containers.add(BuilderUtils
-// .newContainer(containerId, BuilderUtils.newNodeId("host"
-// + containerId.getId(), 2345),
-// "host" + containerId.getId() + ":5678", req
-// .getCapability(), req.getPriority(), null));
-// }
-// }
-//
-// AMResponse amResponse = Records.newRecord(AMResponse.class);
-// amResponse.setAllocatedContainers(containers);
-// amResponse.setResponseId(request.getResponseId() + 1);
-// response.setAMResponse(amResponse);
-// response.setNumClusterNodes(350);
-// return response;
-// }
-// };
-// }
-// };
-// }
-// });
-// }
-//
-// @Test
-// public void benchmark2() throws Exception {
-// int maps = 100; // Adjust for benchmarking, start with a couple of thousands
-// int reduces = 50;
-// int maxConcurrentRunningTasks = 500;
-//
-// System.out.println("Running benchmark with throttled running tasks with " +
-// "maxConcurrentRunningTasks:" + maxConcurrentRunningTasks +
-// " maps:" + maps + " reduces:" + reduces);
-// run(new ThrottledMRApp(maps, reduces, maxConcurrentRunningTasks));
-// }
-//
-// public static void main(String[] args) throws Exception {
-// MRAppBenchmark benchmark = new MRAppBenchmark();
-// benchmark.benchmark1();
-// benchmark.benchmark2();
-// }
-//
-//}
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app2;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.v2.api.records.JobState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.app2.client.ClientService;
+import org.apache.hadoop.mapreduce.v2.app2.job.Job;
+import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptContainerAssignedEvent;
+import org.apache.hadoop.mapreduce.v2.app2.rm.ContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app2.rm.ContainerAllocatorEvent;
+import org.apache.hadoop.mapreduce.v2.app2.rm.RMContainerAllocator;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.junit.Test;
+
+public class MRAppBenchmark {
+
+ private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+
+ /**
+ * Runs memory and time benchmark with Mock MRApp.
+ */
+ public void run(MRApp app) throws Exception {
+ Logger rootLogger = LogManager.getRootLogger();
+ rootLogger.setLevel(Level.WARN);
+ long startTime = System.currentTimeMillis();
+ Job job = app.submit(new Configuration());
+ while (!job.getReport().getJobState().equals(JobState.SUCCEEDED)) {
+ printStat(job, startTime);
+ Thread.sleep(2000);
+ }
+ printStat(job, startTime);
+ }
+
+ private void printStat(Job job, long startTime) throws Exception {
+ long currentTime = System.currentTimeMillis();
+ Runtime.getRuntime().gc();
+ long mem = Runtime.getRuntime().totalMemory()
+ - Runtime.getRuntime().freeMemory();
+ System.out.println("JobState:" + job.getState() +
+ " CompletedMaps:" + job.getCompletedMaps() +
+ " CompletedReduces:" + job.getCompletedReduces() +
+ " Memory(total-free)(KB):" + mem/1024 +
+ " ElapsedTime(ms):" + (currentTime - startTime));
+ }
+
+ //Throttles the maximum number of concurrent running tasks.
+ //This affects the memory requirement since
+ //org.apache.hadoop.mapred.MapTask/ReduceTask is loaded in memory for all
+ //running task and discarded once the task is launched.
+ static class ThrottledMRApp extends MRApp {
+
+ int maxConcurrentRunningTasks;
+ volatile int concurrentRunningTasks;
+ ThrottledMRApp(int maps, int reduces, int maxConcurrentRunningTasks) {
+ super(maps, reduces, true, "ThrottledMRApp", true);
+ this.maxConcurrentRunningTasks = maxConcurrentRunningTasks;
+ }
+
+ @Override
+ protected void attemptLaunched(TaskAttemptId attemptID) {
+ super.attemptLaunched(attemptID);
+ //the task is launched and sends done immediately
+ concurrentRunningTasks--;
+ }
+
+ @Override
+ protected ContainerAllocator createContainerAllocator(
+ ClientService clientService, AppContext context) {
+ return new ThrottledContainerAllocator();
+ }
+
+ class ThrottledContainerAllocator extends AbstractService
+ implements ContainerAllocator {
+ private int containerCount;
+ private Thread thread;
+ private BlockingQueue<ContainerAllocatorEvent> eventQueue =
+ new LinkedBlockingQueue<ContainerAllocatorEvent>();
+ public ThrottledContainerAllocator() {
+ super("ThrottledContainerAllocator");
+ }
+ @Override
+ public void handle(ContainerAllocatorEvent event) {
+ try {
+ eventQueue.put(event);
+ } catch (InterruptedException e) {
+ throw new YarnException(e);
+ }
+ }
+ @Override
+ public void start() {
+ thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ ContainerAllocatorEvent event = null;
+ while (!Thread.currentThread().isInterrupted()) {
+ try {
+ if (concurrentRunningTasks < maxConcurrentRunningTasks) {
+ event = eventQueue.take();
+ ContainerId cId =
+ recordFactory.newRecordInstance(ContainerId.class);
+ cId.setApplicationAttemptId(
+ getContext().getApplicationAttemptId());
+ cId.setId(containerCount++);
+ //System.out.println("Allocating " + containerCount);
+
+ Container container =
+ recordFactory.newRecordInstance(Container.class);
+ container.setId(cId);
+ NodeId nodeId = recordFactory.newRecordInstance(NodeId.class);
+ nodeId.setHost("dummy");
+ nodeId.setPort(1234);
+ container.setNodeId(nodeId);
+ container.setContainerToken(null);
+ container.setNodeHttpAddress("localhost:8042");
+ getContext().getEventHandler()
+ .handle(
+ new TaskAttemptContainerAssignedEvent(event
+ .getAttemptID(), container, null));
+ concurrentRunningTasks++;
+ } else {
+ Thread.sleep(1000);
+ }
+ } catch (InterruptedException e) {
+ System.out.println("Returning, interrupted");
+ return;
+ }
+ }
+ }
+ });
+ thread.start();
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ thread.interrupt();
+ super.stop();
+ }
+ }
+ }
+
+ @Test
+ public void benchmark1() throws Exception {
+ int maps = 100; // Adjust for benchmarking. Start with thousands.
+ int reduces = 0;
+ System.out.println("Running benchmark with maps:"+maps +
+ " reduces:"+reduces);
+ run(new MRApp(maps, reduces, true, this.getClass().getName(), true) {
+
+ @Override
+ protected ContainerAllocator createContainerAllocator(
+ ClientService clientService, AppContext context) {
+ return new RMContainerAllocator(clientService, context) {
+ @Override
+ protected AMRMProtocol createSchedulerProxy() {
+ return new AMRMProtocol() {
+
+ @Override
+ public RegisterApplicationMasterResponse
+ registerApplicationMaster(
+ RegisterApplicationMasterRequest request)
+ throws YarnRemoteException {
+ RegisterApplicationMasterResponse response =
+ Records.newRecord(RegisterApplicationMasterResponse.class);
+ response.setMinimumResourceCapability(BuilderUtils
+ .newResource(1024));
+ response.setMaximumResourceCapability(BuilderUtils
+ .newResource(10240));
+ return response;
+ }
+
+ @Override
+ public FinishApplicationMasterResponse finishApplicationMaster(
+ FinishApplicationMasterRequest request)
+ throws YarnRemoteException {
+ FinishApplicationMasterResponse response =
+ Records.newRecord(FinishApplicationMasterResponse.class);
+ return response;
+ }
+
+ @Override
+ public AllocateResponse allocate(AllocateRequest request)
+ throws YarnRemoteException {
+
+ AllocateResponse response =
+ Records.newRecord(AllocateResponse.class);
+ List<ResourceRequest> askList = request.getAskList();
+ List<Container> containers = new ArrayList<Container>();
+ for (ResourceRequest req : askList) {
+ if (req.getHostName() != "*") {
+ continue;
+ }
+ int numContainers = req.getNumContainers();
+ for (int i = 0; i < numContainers; i++) {
+ ContainerId containerId =
+ BuilderUtils.newContainerId(
+ request.getApplicationAttemptId(),
+ request.getResponseId() + i);
+ containers.add(BuilderUtils
+ .newContainer(containerId, BuilderUtils.newNodeId("host"
+ + containerId.getId(), 2345),
+ "host" + containerId.getId() + ":5678", req
+ .getCapability(), req.getPriority(), null));
+ }
+ }
+
+ AMResponse amResponse = Records.newRecord(AMResponse.class);
+ amResponse.setAllocatedContainers(containers);
+ amResponse.setResponseId(request.getResponseId() + 1);
+ response.setAMResponse(amResponse);
+ response.setNumClusterNodes(350);
+ return response;
+ }
+ };
+ }
+ };
+ }
+ });
+ }
+
+ @Test
+ public void benchmark2() throws Exception {
+ int maps = 100; // Adjust for benchmarking, start with a couple of thousands
+ int reduces = 50;
+ int maxConcurrentRunningTasks = 500;
+
+ System.out.println("Running benchmark with throttled running tasks with " +
+ "maxConcurrentRunningTasks:" + maxConcurrentRunningTasks +
+ " maps:" + maps + " reduces:" + reduces);
+ run(new ThrottledMRApp(maps, reduces, maxConcurrentRunningTasks));
+ }
+
+ public static void main(String[] args) throws Exception {
+ MRAppBenchmark benchmark = new MRAppBenchmark();
+ benchmark.benchmark1();
+ benchmark.benchmark2();
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/job/impl/TestTaskAttempt.java
index 9e192e9..ff4cefe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/job/impl/TestTaskAttempt.java
@@ -1,650 +1,650 @@
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.hadoop.mapreduce.v2.app2.job.impl;
-//
-//import static org.junit.Assert.assertEquals;
-//import static org.junit.Assert.assertFalse;
-//import static org.mockito.Mockito.mock;
-//import static org.mockito.Mockito.spy;
-//import static org.mockito.Mockito.times;
-//import static org.mockito.Mockito.verify;
-//import static org.mockito.Mockito.when;
-//
-//import java.io.IOException;
-//import java.net.InetSocketAddress;
-//import java.util.Arrays;
-//import java.util.HashMap;
-//import java.util.Iterator;
-//import java.util.Map;
-//
-//import junit.framework.Assert;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-//import org.apache.hadoop.fs.FileStatus;
-//import org.apache.hadoop.fs.FileSystem;
-//import org.apache.hadoop.fs.Path;
-//import org.apache.hadoop.fs.RawLocalFileSystem;
-//import org.apache.hadoop.io.DataInputByteBuffer;
-//import org.apache.hadoop.io.Text;
-//import org.apache.hadoop.mapred.JobConf;
-//import org.apache.hadoop.mapred.MapTaskAttemptImpl;
-//import org.apache.hadoop.mapred.WrappedJvmID;
-//import org.apache.hadoop.mapreduce.JobCounter;
-//import org.apache.hadoop.mapreduce.MRJobConfig;
-//import org.apache.hadoop.mapreduce.OutputCommitter;
-//import org.apache.hadoop.mapreduce.TypeConverter;
-//import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
-//import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletion;
-//import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-//import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
-//import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-//import org.apache.hadoop.mapreduce.v2.api.records.JobState;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-//import org.apache.hadoop.mapreduce.v2.app2.AppContext;
-//import org.apache.hadoop.mapreduce.v2.app2.ControlledClock;
-//import org.apache.hadoop.mapreduce.v2.app2.MRApp;
-//import org.apache.hadoop.mapreduce.v2.app2.TaskAttemptListener;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Job;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Task;
-//import org.apache.hadoop.mapreduce.v2.app2.job.TaskAttempt;
-//import org.apache.hadoop.mapreduce.v2.app2.job.event.JobEvent;
-//import org.apache.hadoop.mapreduce.v2.app2.job.event.JobEventType;
-//import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptContainerAssignedEvent;
-//import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptContainerLaunchedEvent;
-//import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptDiagnosticsUpdateEvent;
-//import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptEvent;
-//import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptEventType;
-//import org.apache.hadoop.mapreduce.v2.app2.rm.ContainerRequestEvent;
-//import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
-//import org.apache.hadoop.security.Credentials;
-//import org.apache.hadoop.security.UserGroupInformation;
-//import org.apache.hadoop.security.token.Token;
-//import org.apache.hadoop.security.token.TokenIdentifier;
-//import org.apache.hadoop.yarn.Clock;
-//import org.apache.hadoop.yarn.ClusterInfo;
-//import org.apache.hadoop.yarn.SystemClock;
-//import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-//import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-//import org.apache.hadoop.yarn.api.records.ApplicationId;
-//import org.apache.hadoop.yarn.api.records.Container;
-//import org.apache.hadoop.yarn.api.records.ContainerId;
-//import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-//import org.apache.hadoop.yarn.api.records.NodeId;
-//import org.apache.hadoop.yarn.api.records.Resource;
-//import org.apache.hadoop.yarn.event.Event;
-//import org.apache.hadoop.yarn.event.EventHandler;
-//import org.apache.hadoop.yarn.util.BuilderUtils;
-//import org.junit.Test;
-//import org.mockito.ArgumentCaptor;
-//
-//@SuppressWarnings({"unchecked", "rawtypes"})
-//public class TestTaskAttempt{
-// @Test
-// public void testAttemptContainerRequest() throws Exception {
-// //WARNING: This test must run first. This is because there is an
-// // optimization where the credentials passed in are cached statically so
-// // they do not need to be recomputed when creating a new
-// // ContainerLaunchContext. if other tests run first this code will cache
-// // their credentials and this test will fail trying to look for the
-// // credentials it inserted in.
-// final Text SECRET_KEY_ALIAS = new Text("secretkeyalias");
-// final byte[] SECRET_KEY = ("secretkey").getBytes();
-// Map<ApplicationAccessType, String> acls =
-// new HashMap<ApplicationAccessType, String>(1);
-// acls.put(ApplicationAccessType.VIEW_APP, "otheruser");
-// ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
-// JobId jobId = MRBuilderUtils.newJobId(appId, 1);
-// TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
-// Path jobFile = mock(Path.class);
-//
-// EventHandler eventHandler = mock(EventHandler.class);
-// TaskAttemptListener taListener = mock(TaskAttemptListener.class);
-// when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
-//
-// JobConf jobConf = new JobConf();
-// jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
-// jobConf.setBoolean("fs.file.impl.disable.cache", true);
-// jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
-//
-// // setup UGI for security so tokens and keys are preserved
-// jobConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
-// UserGroupInformation.setConfiguration(jobConf);
-//
-// Credentials credentials = new Credentials();
-// credentials.addSecretKey(SECRET_KEY_ALIAS, SECRET_KEY);
-// Token<JobTokenIdentifier> jobToken = new Token<JobTokenIdentifier>(
-// ("tokenid").getBytes(), ("tokenpw").getBytes(),
-// new Text("tokenkind"), new Text("tokenservice"));
-//
-// TaskAttemptImpl taImpl =
-// new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
-// mock(TaskSplitMetaInfo.class), jobConf, taListener,
-// mock(OutputCommitter.class), jobToken, credentials,
-// new SystemClock(), null);
-//
-// jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, taImpl.getID().toString());
-// ContainerId containerId = BuilderUtils.newContainerId(1, 1, 1, 1);
-//
-// ContainerLaunchContext launchCtx =
-// TaskAttemptImpl.createContainerLaunchContext(acls, containerId,
-// jobConf, jobToken, taImpl.createRemoteTask(),
-// TypeConverter.fromYarn(jobId), mock(Resource.class),
-// mock(WrappedJvmID.class), taListener,
-// credentials);
-//
-// Assert.assertEquals("ACLs mismatch", acls, launchCtx.getApplicationACLs());
-// Credentials launchCredentials = new Credentials();
-//
-// DataInputByteBuffer dibb = new DataInputByteBuffer();
-// dibb.reset(launchCtx.getContainerTokens());
-// launchCredentials.readTokenStorageStream(dibb);
-//
-// // verify all tokens specified for the task attempt are in the launch context
-// for (Token<? extends TokenIdentifier> token : credentials.getAllTokens()) {
-// Token<? extends TokenIdentifier> launchToken =
-// launchCredentials.getToken(token.getService());
-// Assert.assertNotNull("Token " + token.getService() + " is missing",
-// launchToken);
-// Assert.assertEquals("Token " + token.getService() + " mismatch",
-// token, launchToken);
-// }
-//
-// // verify the secret key is in the launch context
-// Assert.assertNotNull("Secret key missing",
-// launchCredentials.getSecretKey(SECRET_KEY_ALIAS));
-// Assert.assertTrue("Secret key mismatch", Arrays.equals(SECRET_KEY,
-// launchCredentials.getSecretKey(SECRET_KEY_ALIAS)));
-// }
-//
-// static public class StubbedFS extends RawLocalFileSystem {
-// @Override
-// public FileStatus getFileStatus(Path f) throws IOException {
-// return new FileStatus(1, false, 1, 1, 1, f);
-// }
-// }
-//
-// @Test
-// public void testMRAppHistoryForMap() throws Exception {
-// MRApp app = new FailingAttemptsMRApp(1, 0);
-// testMRAppHistory(app);
-// }
-//
-// @Test
-// public void testMRAppHistoryForReduce() throws Exception {
-// MRApp app = new FailingAttemptsMRApp(0, 1);
-// testMRAppHistory(app);
-// }
-//
-// @Test
-// public void testSingleRackRequest() throws Exception {
-// TaskAttemptImpl.RequestContainerTransition rct =
-// new TaskAttemptImpl.RequestContainerTransition(false);
-//
-// EventHandler eventHandler = mock(EventHandler.class);
-// String[] hosts = new String[3];
-// hosts[0] = "host1";
-// hosts[1] = "host2";
-// hosts[2] = "host3";
-// TaskSplitMetaInfo splitInfo =
-// new TaskSplitMetaInfo(hosts, 0, 128 * 1024 * 1024l);
-//
-// TaskAttemptImpl mockTaskAttempt =
-// createMapTaskAttemptImplForTest(eventHandler, splitInfo);
-// TaskAttemptEvent mockTAEvent = mock(TaskAttemptEvent.class);
-//
-// rct.transition(mockTaskAttempt, mockTAEvent);
-//
-// ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
-// verify(eventHandler, times(2)).handle(arg.capture());
-// if (!(arg.getAllValues().get(1) instanceof ContainerRequestEvent)) {
-// Assert.fail("Second Event not of type ContainerRequestEvent");
-// }
-// ContainerRequestEvent cre =
-// (ContainerRequestEvent) arg.getAllValues().get(1);
-// String[] requestedRacks = cre.getRacks();
-// //Only a single occurrence of /DefaultRack
-// assertEquals(1, requestedRacks.length);
-// }
-//
-// @Test
-// public void testHostResolveAttempt() throws Exception {
-// TaskAttemptImpl.RequestContainerTransition rct =
-// new TaskAttemptImpl.RequestContainerTransition(false);
-//
-// EventHandler eventHandler = mock(EventHandler.class);
-// String[] hosts = new String[3];
-// hosts[0] = "192.168.1.1";
-// hosts[1] = "host2";
-// hosts[2] = "host3";
-// TaskSplitMetaInfo splitInfo =
-// new TaskSplitMetaInfo(hosts, 0, 128 * 1024 * 1024l);
-//
-// TaskAttemptImpl mockTaskAttempt =
-// createMapTaskAttemptImplForTest(eventHandler, splitInfo);
-// TaskAttemptImpl spyTa = spy(mockTaskAttempt);
-// when(spyTa.resolveHost(hosts[0])).thenReturn("host1");
-//
-// TaskAttemptEvent mockTAEvent = mock(TaskAttemptEvent.class);
-// rct.transition(spyTa, mockTAEvent);
-// verify(spyTa).resolveHost(hosts[0]);
-// ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
-// verify(eventHandler, times(2)).handle(arg.capture());
-// if (!(arg.getAllValues().get(1) instanceof ContainerRequestEvent)) {
-// Assert.fail("Second Event not of type ContainerRequestEvent");
-// }
-// Map<String, Boolean> expected = new HashMap<String, Boolean>();
-// expected.put("host1", true);
-// expected.put("host2", true);
-// expected.put("host3", true);
-// ContainerRequestEvent cre =
-// (ContainerRequestEvent) arg.getAllValues().get(1);
-// String[] requestedHosts = cre.getHosts();
-// for (String h : requestedHosts) {
-// expected.remove(h);
-// }
-// assertEquals(0, expected.size());
-// }
-//
-// @Test
-// public void testSlotMillisCounterUpdate() throws Exception {
-// verifySlotMillis(2048, 2048, 1024);
-// verifySlotMillis(2048, 1024, 1024);
-// verifySlotMillis(10240, 1024, 2048);
-// }
-//
-// public void verifySlotMillis(int mapMemMb, int reduceMemMb,
-// int minContainerSize) throws Exception {
-// Clock actualClock = new SystemClock();
-// ControlledClock clock = new ControlledClock(actualClock);
-// clock.setTime(10);
-// MRApp app =
-// new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock);
-// Configuration conf = new Configuration();
-// conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
-// conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
-// app.setClusterInfo(new ClusterInfo(BuilderUtils
-// .newResource(minContainerSize), BuilderUtils.newResource(10240)));
-//
-// Job job = app.submit(conf);
-// app.waitForState(job, JobState.RUNNING);
-// Map<TaskId, Task> tasks = job.getTasks();
-// Assert.assertEquals("Num tasks is not correct", 2, tasks.size());
-// Iterator<Task> taskIter = tasks.values().iterator();
-// Task mTask = taskIter.next();
-// app.waitForState(mTask, TaskState.RUNNING);
-// Task rTask = taskIter.next();
-// app.waitForState(rTask, TaskState.RUNNING);
-// Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts();
-// Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size());
-// Map<TaskAttemptId, TaskAttempt> rAttempts = rTask.getAttempts();
-// Assert.assertEquals("Num attempts is not correct", 1, rAttempts.size());
-// TaskAttempt mta = mAttempts.values().iterator().next();
-// TaskAttempt rta = rAttempts.values().iterator().next();
-// app.waitForState(mta, TaskAttemptState.RUNNING);
-// app.waitForState(rta, TaskAttemptState.RUNNING);
-//
-// clock.setTime(11);
-// app.getContext()
-// .getEventHandler()
-// .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
-// app.getContext()
-// .getEventHandler()
-// .handle(new TaskAttemptEvent(rta.getID(), TaskAttemptEventType.TA_DONE));
-// app.waitForState(job, JobState.SUCCEEDED);
-// Assert.assertEquals(mta.getFinishTime(), 11);
-// Assert.assertEquals(mta.getLaunchTime(), 10);
-// Assert.assertEquals(rta.getFinishTime(), 11);
-// Assert.assertEquals(rta.getLaunchTime(), 10);
-// Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
-// job.getAllCounters().findCounter(JobCounter.SLOTS_MILLIS_MAPS)
-// .getValue());
-// Assert.assertEquals(
-// (int) Math.ceil((float) reduceMemMb / minContainerSize), job
-// .getAllCounters().findCounter(JobCounter.SLOTS_MILLIS_REDUCES)
-// .getValue());
-// }
-//
-// private TaskAttemptImpl createMapTaskAttemptImplForTest(
-// EventHandler eventHandler, TaskSplitMetaInfo taskSplitMetaInfo) {
-// Clock clock = new SystemClock();
-// return createMapTaskAttemptImplForTest(eventHandler, taskSplitMetaInfo, clock);
-// }
-//
-// private TaskAttemptImpl createMapTaskAttemptImplForTest(
-// EventHandler eventHandler, TaskSplitMetaInfo taskSplitMetaInfo, Clock clock) {
-// ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
-// JobId jobId = MRBuilderUtils.newJobId(appId, 1);
-// TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
-// TaskAttemptListener taListener = mock(TaskAttemptListener.class);
-// Path jobFile = mock(Path.class);
-// JobConf jobConf = new JobConf();
-// OutputCommitter outputCommitter = mock(OutputCommitter.class);
-// TaskAttemptImpl taImpl =
-// new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
-// taskSplitMetaInfo, jobConf, taListener, outputCommitter, null,
-// null, clock, null);
-// return taImpl;
-// }
-//
-// private void testMRAppHistory(MRApp app) throws Exception {
-// Configuration conf = new Configuration();
-// Job job = app.submit(conf);
-// app.waitForState(job, JobState.FAILED);
-// Map<TaskId, Task> tasks = job.getTasks();
-//
-// Assert.assertEquals("Num tasks is not correct", 1, tasks.size());
-// Task task = tasks.values().iterator().next();
-// Assert.assertEquals("Task state not correct", TaskState.FAILED, task
-// .getReport().getTaskState());
-// Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next()
-// .getAttempts();
-// Assert.assertEquals("Num attempts is not correct", 4, attempts.size());
-//
-// Iterator<TaskAttempt> it = attempts.values().iterator();
-// TaskAttemptReport report = it.next().getReport();
-// Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
-// report.getTaskAttemptState());
-// Assert.assertEquals("Diagnostic Information is not Correct",
-// "Test Diagnostic Event", report.getDiagnosticInfo());
-// report = it.next().getReport();
-// Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
-// report.getTaskAttemptState());
-// }
-//
-// static class FailingAttemptsMRApp extends MRApp {
-// FailingAttemptsMRApp(int maps, int reduces) {
-// super(maps, reduces, true, "FailingAttemptsMRApp", true);
-// }
-//
-// @Override
-// protected void attemptLaunched(TaskAttemptId attemptID) {
-// getContext().getEventHandler().handle(
-// new TaskAttemptDiagnosticsUpdateEvent(attemptID,
-// "Test Diagnostic Event"));
-// getContext().getEventHandler().handle(
-// new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
-// }
-//
-// protected EventHandler<JobHistoryEvent> createJobHistoryHandler(
-// AppContext context) {
-// return new EventHandler<JobHistoryEvent>() {
-// @Override
-// public void handle(JobHistoryEvent event) {
-// if (event.getType() == org.apache.hadoop.mapreduce.jobhistory.EventType.MAP_ATTEMPT_FAILED) {
-// TaskAttemptUnsuccessfulCompletion datum = (TaskAttemptUnsuccessfulCompletion) event
-// .getHistoryEvent().getDatum();
-// Assert.assertEquals("Diagnostic Information is not Correct",
-// "Test Diagnostic Event", datum.get(8).toString());
-// }
-// }
-// };
-// }
-// }
-//
-// @Test
-// public void testLaunchFailedWhileKilling() throws Exception {
-// ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
-// ApplicationAttemptId appAttemptId =
-// BuilderUtils.newApplicationAttemptId(appId, 0);
-// JobId jobId = MRBuilderUtils.newJobId(appId, 1);
-// TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
-// TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
-// Path jobFile = mock(Path.class);
-//
-// MockEventHandler eventHandler = new MockEventHandler();
-// TaskAttemptListener taListener = mock(TaskAttemptListener.class);
-// when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
-//
-// JobConf jobConf = new JobConf();
-// jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
-// jobConf.setBoolean("fs.file.impl.disable.cache", true);
-// jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
-// jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
-//
-// TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
-// when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
-//
-// TaskAttemptImpl taImpl =
-// new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
-// splits, jobConf, taListener,
-// mock(OutputCommitter.class), mock(Token.class), new Credentials(),
-// new SystemClock(), null);
-//
-// NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
-// ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
-// Container container = mock(Container.class);
-// when(container.getId()).thenReturn(contId);
-// when(container.getNodeId()).thenReturn(nid);
-//
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_SCHEDULE));
-// taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
-// container, mock(Map.class)));
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_KILL));
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_CONTAINER_CLEANED));
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED));
-// assertFalse(eventHandler.internalError);
-// }
-//
-// @Test
-// public void testContainerCleanedWhileRunning() throws Exception {
-// ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
-// ApplicationAttemptId appAttemptId =
-// BuilderUtils.newApplicationAttemptId(appId, 0);
-// JobId jobId = MRBuilderUtils.newJobId(appId, 1);
-// TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
-// TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
-// Path jobFile = mock(Path.class);
-//
-// MockEventHandler eventHandler = new MockEventHandler();
-// TaskAttemptListener taListener = mock(TaskAttemptListener.class);
-// when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
-//
-// JobConf jobConf = new JobConf();
-// jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
-// jobConf.setBoolean("fs.file.impl.disable.cache", true);
-// jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
-// jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
-//
-// TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
-// when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
-//
-// AppContext appCtx = mock(AppContext.class);
-// ClusterInfo clusterInfo = mock(ClusterInfo.class);
-// Resource resource = mock(Resource.class);
-// when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
-// when(clusterInfo.getMinContainerCapability()).thenReturn(resource);
-// when(resource.getMemory()).thenReturn(1024);
-//
-// TaskAttemptImpl taImpl =
-// new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
-// splits, jobConf, taListener,
-// mock(OutputCommitter.class), mock(Token.class), new Credentials(),
-// new SystemClock(), appCtx);
-//
-// NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
-// ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
-// Container container = mock(Container.class);
-// when(container.getId()).thenReturn(contId);
-// when(container.getNodeId()).thenReturn(nid);
-// when(container.getNodeHttpAddress()).thenReturn("localhost:0");
-//
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_SCHEDULE));
-// taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
-// container, mock(Map.class)));
-// taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
-// assertEquals("Task attempt is not in running state", taImpl.getState(),
-// TaskAttemptState.RUNNING);
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_CONTAINER_CLEANED));
-// assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED",
-// eventHandler.internalError);
-// }
-//
-// @Test
-// public void testContainerCleanedWhileCommitting() throws Exception {
-// ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
-// ApplicationAttemptId appAttemptId =
-// BuilderUtils.newApplicationAttemptId(appId, 0);
-// JobId jobId = MRBuilderUtils.newJobId(appId, 1);
-// TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
-// TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
-// Path jobFile = mock(Path.class);
-//
-// MockEventHandler eventHandler = new MockEventHandler();
-// TaskAttemptListener taListener = mock(TaskAttemptListener.class);
-// when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
-//
-// JobConf jobConf = new JobConf();
-// jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
-// jobConf.setBoolean("fs.file.impl.disable.cache", true);
-// jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
-// jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
-//
-// TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
-// when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
-//
-// AppContext appCtx = mock(AppContext.class);
-// ClusterInfo clusterInfo = mock(ClusterInfo.class);
-// Resource resource = mock(Resource.class);
-// when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
-// when(clusterInfo.getMinContainerCapability()).thenReturn(resource);
-// when(resource.getMemory()).thenReturn(1024);
-//
-// TaskAttemptImpl taImpl =
-// new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
-// splits, jobConf, taListener,
-// mock(OutputCommitter.class), mock(Token.class), new Credentials(),
-// new SystemClock(), appCtx);
-//
-// NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
-// ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
-// Container container = mock(Container.class);
-// when(container.getId()).thenReturn(contId);
-// when(container.getNodeId()).thenReturn(nid);
-// when(container.getNodeHttpAddress()).thenReturn("localhost:0");
-//
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_SCHEDULE));
-// taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
-// container, mock(Map.class)));
-// taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_COMMIT_PENDING));
-//
-// assertEquals("Task attempt is not in commit pending state", taImpl.getState(),
-// TaskAttemptState.COMMIT_PENDING);
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_CONTAINER_CLEANED));
-// assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED",
-// eventHandler.internalError);
-// }
-//
-// @Test
-// public void testDoubleTooManyFetchFailure() throws Exception {
-// ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
-// ApplicationAttemptId appAttemptId =
-// BuilderUtils.newApplicationAttemptId(appId, 0);
-// JobId jobId = MRBuilderUtils.newJobId(appId, 1);
-// TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
-// TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
-// Path jobFile = mock(Path.class);
-//
-// MockEventHandler eventHandler = new MockEventHandler();
-// TaskAttemptListener taListener = mock(TaskAttemptListener.class);
-// when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
-//
-// JobConf jobConf = new JobConf();
-// jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
-// jobConf.setBoolean("fs.file.impl.disable.cache", true);
-// jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
-// jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
-//
-// TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
-// when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
-//
-// AppContext appCtx = mock(AppContext.class);
-// ClusterInfo clusterInfo = mock(ClusterInfo.class);
-// Resource resource = mock(Resource.class);
-// when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
-// when(clusterInfo.getMinContainerCapability()).thenReturn(resource);
-// when(resource.getMemory()).thenReturn(1024);
-//
-// TaskAttemptImpl taImpl =
-// new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
-// splits, jobConf, taListener,
-// mock(OutputCommitter.class), mock(Token.class), new Credentials(),
-// new SystemClock(), appCtx);
-//
-// NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
-// ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
-// Container container = mock(Container.class);
-// when(container.getId()).thenReturn(contId);
-// when(container.getNodeId()).thenReturn(nid);
-// when(container.getNodeHttpAddress()).thenReturn("localhost:0");
-//
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_SCHEDULE));
-// taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
-// container, mock(Map.class)));
-// taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_DONE));
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_CONTAINER_CLEANED));
-//
-// assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
-// TaskAttemptState.SUCCEEDED);
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
-// assertEquals("Task attempt is not in FAILED state", taImpl.getState(),
-// TaskAttemptState.FAILED);
-// taImpl.handle(new TaskAttemptEvent(attemptId,
-// TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
-// assertEquals("Task attempt is not in FAILED state, still", taImpl.getState(),
-// TaskAttemptState.FAILED);
-// assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED",
-// eventHandler.internalError);
-// }
-//
-// public static class MockEventHandler implements EventHandler {
-// public boolean internalError;
-//
-// @Override
-// public void handle(Event event) {
-// if (event instanceof JobEvent) {
-// JobEvent je = ((JobEvent) event);
-// if (JobEventType.INTERNAL_ERROR == je.getType()) {
-// internalError = true;
-// }
-// }
-// }
-//
-// };
-//}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.app2.job.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapTaskAttemptImpl;
+import org.apache.hadoop.mapred.WrappedJvmID;
+import org.apache.hadoop.mapreduce.JobCounter;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
+import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletion;
+import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
+import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.JobState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app2.AppContext;
+import org.apache.hadoop.mapreduce.v2.app2.ControlledClock;
+import org.apache.hadoop.mapreduce.v2.app2.MRApp;
+import org.apache.hadoop.mapreduce.v2.app2.TaskAttemptListener;
+import org.apache.hadoop.mapreduce.v2.app2.job.Job;
+import org.apache.hadoop.mapreduce.v2.app2.job.Task;
+import org.apache.hadoop.mapreduce.v2.app2.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app2.job.event.JobEvent;
+import org.apache.hadoop.mapreduce.v2.app2.job.event.JobEventType;
+import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptContainerAssignedEvent;
+import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptContainerLaunchedEvent;
+import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app2.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app2.rm.ContainerRequestEvent;
+import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.ClusterInfo;
+import org.apache.hadoop.yarn.SystemClock;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+@SuppressWarnings({"unchecked", "rawtypes"})
+public class TestTaskAttempt{
+ @Test
+ public void testAttemptContainerRequest() throws Exception {
+ //WARNING: This test must run first. This is because there is an
+ // optimization where the credentials passed in are cached statically so
+ // they do not need to be recomputed when creating a new
+ // ContainerLaunchContext. if other tests run first this code will cache
+ // their credentials and this test will fail trying to look for the
+ // credentials it inserted in.
+ final Text SECRET_KEY_ALIAS = new Text("secretkeyalias");
+ final byte[] SECRET_KEY = ("secretkey").getBytes();
+ Map<ApplicationAccessType, String> acls =
+ new HashMap<ApplicationAccessType, String>(1);
+ acls.put(ApplicationAccessType.VIEW_APP, "otheruser");
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
+ JobId jobId = MRBuilderUtils.newJobId(appId, 1);
+ TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
+ Path jobFile = mock(Path.class);
+
+ EventHandler eventHandler = mock(EventHandler.class);
+ TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
+
+ JobConf jobConf = new JobConf();
+ jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+ jobConf.setBoolean("fs.file.impl.disable.cache", true);
+ jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
+
+ // setup UGI for security so tokens and keys are preserved
+ jobConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(jobConf);
+
+ Credentials credentials = new Credentials();
+ credentials.addSecretKey(SECRET_KEY_ALIAS, SECRET_KEY);
+ Token<JobTokenIdentifier> jobToken = new Token<JobTokenIdentifier>(
+ ("tokenid").getBytes(), ("tokenpw").getBytes(),
+ new Text("tokenkind"), new Text("tokenservice"));
+
+ TaskAttemptImpl taImpl =
+ new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
+ mock(TaskSplitMetaInfo.class), jobConf, taListener,
+ mock(OutputCommitter.class), jobToken, credentials,
+ new SystemClock(), null);
+
+ jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, taImpl.getID().toString());
+ ContainerId containerId = BuilderUtils.newContainerId(1, 1, 1, 1);
+
+ ContainerLaunchContext launchCtx =
+ TaskAttemptImpl.createContainerLaunchContext(acls, containerId,
+ jobConf, jobToken, taImpl.createRemoteTask(),
+ TypeConverter.fromYarn(jobId), mock(Resource.class),
+ mock(WrappedJvmID.class), taListener,
+ credentials);
+
+ Assert.assertEquals("ACLs mismatch", acls, launchCtx.getApplicationACLs());
+ Credentials launchCredentials = new Credentials();
+
+ DataInputByteBuffer dibb = new DataInputByteBuffer();
+ dibb.reset(launchCtx.getContainerTokens());
+ launchCredentials.readTokenStorageStream(dibb);
+
+ // verify all tokens specified for the task attempt are in the launch context
+ for (Token<? extends TokenIdentifier> token : credentials.getAllTokens()) {
+ Token<? extends TokenIdentifier> launchToken =
+ launchCredentials.getToken(token.getService());
+ Assert.assertNotNull("Token " + token.getService() + " is missing",
+ launchToken);
+ Assert.assertEquals("Token " + token.getService() + " mismatch",
+ token, launchToken);
+ }
+
+ // verify the secret key is in the launch context
+ Assert.assertNotNull("Secret key missing",
+ launchCredentials.getSecretKey(SECRET_KEY_ALIAS));
+ Assert.assertTrue("Secret key mismatch", Arrays.equals(SECRET_KEY,
+ launchCredentials.getSecretKey(SECRET_KEY_ALIAS)));
+ }
+
+ static public class StubbedFS extends RawLocalFileSystem {
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ return new FileStatus(1, false, 1, 1, 1, f);
+ }
+ }
+
+ @Test
+ public void testMRAppHistoryForMap() throws Exception {
+ MRApp app = new FailingAttemptsMRApp(1, 0);
+ testMRAppHistory(app);
+ }
+
+ @Test
+ public void testMRAppHistoryForReduce() throws Exception {
+ MRApp app = new FailingAttemptsMRApp(0, 1);
+ testMRAppHistory(app);
+ }
+
+ @Test
+ public void testSingleRackRequest() throws Exception {
+ TaskAttemptImpl.RequestContainerTransition rct =
+ new TaskAttemptImpl.RequestContainerTransition(false);
+
+ EventHandler eventHandler = mock(EventHandler.class);
+ String[] hosts = new String[3];
+ hosts[0] = "host1";
+ hosts[1] = "host2";
+ hosts[2] = "host3";
+ TaskSplitMetaInfo splitInfo =
+ new TaskSplitMetaInfo(hosts, 0, 128 * 1024 * 1024l);
+
+ TaskAttemptImpl mockTaskAttempt =
+ createMapTaskAttemptImplForTest(eventHandler, splitInfo);
+ TaskAttemptEvent mockTAEvent = mock(TaskAttemptEvent.class);
+
+ rct.transition(mockTaskAttempt, mockTAEvent);
+
+ ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(2)).handle(arg.capture());
+ if (!(arg.getAllValues().get(1) instanceof ContainerRequestEvent)) {
+ Assert.fail("Second Event not of type ContainerRequestEvent");
+ }
+ ContainerRequestEvent cre =
+ (ContainerRequestEvent) arg.getAllValues().get(1);
+ String[] requestedRacks = cre.getRacks();
+ //Only a single occurrence of /DefaultRack
+ assertEquals(1, requestedRacks.length);
+ }
+
+ @Test
+ public void testHostResolveAttempt() throws Exception {
+ TaskAttemptImpl.RequestContainerTransition rct =
+ new TaskAttemptImpl.RequestContainerTransition(false);
+
+ EventHandler eventHandler = mock(EventHandler.class);
+ String[] hosts = new String[3];
+ hosts[0] = "192.168.1.1";
+ hosts[1] = "host2";
+ hosts[2] = "host3";
+ TaskSplitMetaInfo splitInfo =
+ new TaskSplitMetaInfo(hosts, 0, 128 * 1024 * 1024l);
+
+ TaskAttemptImpl mockTaskAttempt =
+ createMapTaskAttemptImplForTest(eventHandler, splitInfo);
+ TaskAttemptImpl spyTa = spy(mockTaskAttempt);
+ when(spyTa.resolveHost(hosts[0])).thenReturn("host1");
+
+ TaskAttemptEvent mockTAEvent = mock(TaskAttemptEvent.class);
+ rct.transition(spyTa, mockTAEvent);
+ verify(spyTa).resolveHost(hosts[0]);
+ ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(2)).handle(arg.capture());
+ if (!(arg.getAllValues().get(1) instanceof ContainerRequestEvent)) {
+ Assert.fail("Second Event not of type ContainerRequestEvent");
+ }
+ Map<String, Boolean> expected = new HashMap<String, Boolean>();
+ expected.put("host1", true);
+ expected.put("host2", true);
+ expected.put("host3", true);
+ ContainerRequestEvent cre =
+ (ContainerRequestEvent) arg.getAllValues().get(1);
+ String[] requestedHosts = cre.getHosts();
+ for (String h : requestedHosts) {
+ expected.remove(h);
+ }
+ assertEquals(0, expected.size());
+ }
+
+ @Test
+ public void testSlotMillisCounterUpdate() throws Exception {
+ verifySlotMillis(2048, 2048, 1024);
+ verifySlotMillis(2048, 1024, 1024);
+ verifySlotMillis(10240, 1024, 2048);
+ }
+
+ public void verifySlotMillis(int mapMemMb, int reduceMemMb,
+ int minContainerSize) throws Exception {
+ Clock actualClock = new SystemClock();
+ ControlledClock clock = new ControlledClock(actualClock);
+ clock.setTime(10);
+ MRApp app =
+ new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock);
+ Configuration conf = new Configuration();
+ conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
+ conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
+ app.setClusterInfo(new ClusterInfo(BuilderUtils
+ .newResource(minContainerSize), BuilderUtils.newResource(10240)));
+
+ Job job = app.submit(conf);
+ app.waitForState(job, JobState.RUNNING);
+ Map<TaskId, Task> tasks = job.getTasks();
+ Assert.assertEquals("Num tasks is not correct", 2, tasks.size());
+ Iterator<Task> taskIter = tasks.values().iterator();
+ Task mTask = taskIter.next();
+ app.waitForState(mTask, TaskState.RUNNING);
+ Task rTask = taskIter.next();
+ app.waitForState(rTask, TaskState.RUNNING);
+ Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts();
+ Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size());
+ Map<TaskAttemptId, TaskAttempt> rAttempts = rTask.getAttempts();
+ Assert.assertEquals("Num attempts is not correct", 1, rAttempts.size());
+ TaskAttempt mta = mAttempts.values().iterator().next();
+ TaskAttempt rta = rAttempts.values().iterator().next();
+ app.waitForState(mta, TaskAttemptState.RUNNING);
+ app.waitForState(rta, TaskAttemptState.RUNNING);
+
+ clock.setTime(11);
+ app.getContext()
+ .getEventHandler()
+ .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
+ app.getContext()
+ .getEventHandler()
+ .handle(new TaskAttemptEvent(rta.getID(), TaskAttemptEventType.TA_DONE));
+ app.waitForState(job, JobState.SUCCEEDED);
+ Assert.assertEquals(mta.getFinishTime(), 11);
+ Assert.assertEquals(mta.getLaunchTime(), 10);
+ Assert.assertEquals(rta.getFinishTime(), 11);
+ Assert.assertEquals(rta.getLaunchTime(), 10);
+ Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
+ job.getAllCounters().findCounter(JobCounter.SLOTS_MILLIS_MAPS)
+ .getValue());
+ Assert.assertEquals(
+ (int) Math.ceil((float) reduceMemMb / minContainerSize), job
+ .getAllCounters().findCounter(JobCounter.SLOTS_MILLIS_REDUCES)
+ .getValue());
+ }
+
+ private TaskAttemptImpl createMapTaskAttemptImplForTest(
+ EventHandler eventHandler, TaskSplitMetaInfo taskSplitMetaInfo) {
+ Clock clock = new SystemClock();
+ return createMapTaskAttemptImplForTest(eventHandler, taskSplitMetaInfo, clock);
+ }
+
+ private TaskAttemptImpl createMapTaskAttemptImplForTest(
+ EventHandler eventHandler, TaskSplitMetaInfo taskSplitMetaInfo, Clock clock) {
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
+ JobId jobId = MRBuilderUtils.newJobId(appId, 1);
+ TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
+ TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ Path jobFile = mock(Path.class);
+ JobConf jobConf = new JobConf();
+ OutputCommitter outputCommitter = mock(OutputCommitter.class);
+ TaskAttemptImpl taImpl =
+ new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
+ taskSplitMetaInfo, jobConf, taListener, outputCommitter, null,
+ null, clock, null);
+ return taImpl;
+ }
+
+ private void testMRAppHistory(MRApp app) throws Exception {
+ Configuration conf = new Configuration();
+ Job job = app.submit(conf);
+ app.waitForState(job, JobState.FAILED);
+ Map<TaskId, Task> tasks = job.getTasks();
+
+ Assert.assertEquals("Num tasks is not correct", 1, tasks.size());
+ Task task = tasks.values().iterator().next();
+ Assert.assertEquals("Task state not correct", TaskState.FAILED, task
+ .getReport().getTaskState());
+ Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next()
+ .getAttempts();
+ Assert.assertEquals("Num attempts is not correct", 4, attempts.size());
+
+ Iterator<TaskAttempt> it = attempts.values().iterator();
+ TaskAttemptReport report = it.next().getReport();
+ Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
+ report.getTaskAttemptState());
+ Assert.assertEquals("Diagnostic Information is not Correct",
+ "Test Diagnostic Event", report.getDiagnosticInfo());
+ report = it.next().getReport();
+ Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
+ report.getTaskAttemptState());
+ }
+
+ static class FailingAttemptsMRApp extends MRApp {
+ FailingAttemptsMRApp(int maps, int reduces) {
+ super(maps, reduces, true, "FailingAttemptsMRApp", true);
+ }
+
+ @Override
+ protected void attemptLaunched(TaskAttemptId attemptID) {
+ getContext().getEventHandler().handle(
+ new TaskAttemptDiagnosticsUpdateEvent(attemptID,
+ "Test Diagnostic Event"));
+ getContext().getEventHandler().handle(
+ new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
+ }
+
+ protected EventHandler<JobHistoryEvent> createJobHistoryHandler(
+ AppContext context) {
+ return new EventHandler<JobHistoryEvent>() {
+ @Override
+ public void handle(JobHistoryEvent event) {
+ if (event.getType() == org.apache.hadoop.mapreduce.jobhistory.EventType.MAP_ATTEMPT_FAILED) {
+ TaskAttemptUnsuccessfulCompletion datum = (TaskAttemptUnsuccessfulCompletion) event
+ .getHistoryEvent().getDatum();
+ Assert.assertEquals("Diagnostic Information is not Correct",
+ "Test Diagnostic Event", datum.get(8).toString());
+ }
+ }
+ };
+ }
+ }
+
+ @Test
+ public void testLaunchFailedWhileKilling() throws Exception {
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
+ ApplicationAttemptId appAttemptId =
+ BuilderUtils.newApplicationAttemptId(appId, 0);
+ JobId jobId = MRBuilderUtils.newJobId(appId, 1);
+ TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
+ TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+ Path jobFile = mock(Path.class);
+
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
+
+ JobConf jobConf = new JobConf();
+ jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+ jobConf.setBoolean("fs.file.impl.disable.cache", true);
+ jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
+ jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
+
+ TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
+ when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
+
+ TaskAttemptImpl taImpl =
+ new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
+ splits, jobConf, taListener,
+ mock(OutputCommitter.class), mock(Token.class), new Credentials(),
+ new SystemClock(), null);
+
+ NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
+ ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(contId);
+ when(container.getNodeId()).thenReturn(nid);
+
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_SCHEDULE));
+ taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
+ container, mock(Map.class)));
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_KILL));
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_CONTAINER_CLEANED));
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED));
+ assertFalse(eventHandler.internalError);
+ }
+
+ @Test
+ public void testContainerCleanedWhileRunning() throws Exception {
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
+ ApplicationAttemptId appAttemptId =
+ BuilderUtils.newApplicationAttemptId(appId, 0);
+ JobId jobId = MRBuilderUtils.newJobId(appId, 1);
+ TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
+ TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+ Path jobFile = mock(Path.class);
+
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
+
+ JobConf jobConf = new JobConf();
+ jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+ jobConf.setBoolean("fs.file.impl.disable.cache", true);
+ jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
+ jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
+
+ TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
+ when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
+
+ AppContext appCtx = mock(AppContext.class);
+ ClusterInfo clusterInfo = mock(ClusterInfo.class);
+ Resource resource = mock(Resource.class);
+ when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
+ when(clusterInfo.getMinContainerCapability()).thenReturn(resource);
+ when(resource.getMemory()).thenReturn(1024);
+
+ TaskAttemptImpl taImpl =
+ new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
+ splits, jobConf, taListener,
+ mock(OutputCommitter.class), mock(Token.class), new Credentials(),
+ new SystemClock(), appCtx);
+
+ NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
+ ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(contId);
+ when(container.getNodeId()).thenReturn(nid);
+ when(container.getNodeHttpAddress()).thenReturn("localhost:0");
+
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_SCHEDULE));
+ taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
+ container, mock(Map.class)));
+ taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
+ assertEquals("Task attempt is not in running state", taImpl.getState(),
+ TaskAttemptState.RUNNING);
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_CONTAINER_CLEANED));
+ assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED",
+ eventHandler.internalError);
+ }
+
+ @Test
+ public void testContainerCleanedWhileCommitting() throws Exception {
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
+ ApplicationAttemptId appAttemptId =
+ BuilderUtils.newApplicationAttemptId(appId, 0);
+ JobId jobId = MRBuilderUtils.newJobId(appId, 1);
+ TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
+ TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+ Path jobFile = mock(Path.class);
+
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
+
+ JobConf jobConf = new JobConf();
+ jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+ jobConf.setBoolean("fs.file.impl.disable.cache", true);
+ jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
+ jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
+
+ TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
+ when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
+
+ AppContext appCtx = mock(AppContext.class);
+ ClusterInfo clusterInfo = mock(ClusterInfo.class);
+ Resource resource = mock(Resource.class);
+ when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
+ when(clusterInfo.getMinContainerCapability()).thenReturn(resource);
+ when(resource.getMemory()).thenReturn(1024);
+
+ TaskAttemptImpl taImpl =
+ new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
+ splits, jobConf, taListener,
+ mock(OutputCommitter.class), mock(Token.class), new Credentials(),
+ new SystemClock(), appCtx);
+
+ NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
+ ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(contId);
+ when(container.getNodeId()).thenReturn(nid);
+ when(container.getNodeHttpAddress()).thenReturn("localhost:0");
+
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_SCHEDULE));
+ taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
+ container, mock(Map.class)));
+ taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_COMMIT_PENDING));
+
+ assertEquals("Task attempt is not in commit pending state", taImpl.getState(),
+ TaskAttemptState.COMMIT_PENDING);
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_CONTAINER_CLEANED));
+ assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED",
+ eventHandler.internalError);
+ }
+
+ @Test
+ public void testDoubleTooManyFetchFailure() throws Exception {
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
+ ApplicationAttemptId appAttemptId =
+ BuilderUtils.newApplicationAttemptId(appId, 0);
+ JobId jobId = MRBuilderUtils.newJobId(appId, 1);
+ TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
+ TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+ Path jobFile = mock(Path.class);
+
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));
+
+ JobConf jobConf = new JobConf();
+ jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+ jobConf.setBoolean("fs.file.impl.disable.cache", true);
+ jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
+ jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
+
+ TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
+ when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
+
+ AppContext appCtx = mock(AppContext.class);
+ ClusterInfo clusterInfo = mock(ClusterInfo.class);
+ Resource resource = mock(Resource.class);
+ when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
+ when(clusterInfo.getMinContainerCapability()).thenReturn(resource);
+ when(resource.getMemory()).thenReturn(1024);
+
+ TaskAttemptImpl taImpl =
+ new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
+ splits, jobConf, taListener,
+ mock(OutputCommitter.class), mock(Token.class), new Credentials(),
+ new SystemClock(), appCtx);
+
+ NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
+ ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(contId);
+ when(container.getNodeId()).thenReturn(nid);
+ when(container.getNodeHttpAddress()).thenReturn("localhost:0");
+
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_SCHEDULE));
+ taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
+ container, mock(Map.class)));
+ taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_DONE));
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_CONTAINER_CLEANED));
+
+ assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
+ TaskAttemptState.SUCCEEDED);
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
+ assertEquals("Task attempt is not in FAILED state", taImpl.getState(),
+ TaskAttemptState.FAILED);
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
+ assertEquals("Task attempt is not in FAILED state, still", taImpl.getState(),
+ TaskAttemptState.FAILED);
+ assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED",
+ eventHandler.internalError);
+ }
+
+ public static class MockEventHandler implements EventHandler {
+ public boolean internalError;
+
+ @Override
+ public void handle(Event event) {
+ if (event instanceof JobEvent) {
+ JobEvent je = ((JobEvent) event);
+ if (JobEventType.INTERNAL_ERROR == je.getType()) {
+ internalError = true;
+ }
+ }
+ }
+
+ };
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebApp.java
index 5e199ce..7e177fe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebApp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebApp.java
@@ -1,220 +1,220 @@
-///**
-//* Licensed to the Apache Software Foundation (ASF) under one
-//* or more contributor license agreements. See the NOTICE file
-//* distributed with this work for additional information
-//* regarding copyright ownership. The ASF licenses this file
-//* to you under the Apache License, Version 2.0 (the
-//* "License"); you may not use this file except in compliance
-//* with the License. You may obtain a copy of the License at
-//*
-//* http://www.apache.org/licenses/LICENSE-2.0
-//*
-//* Unless required by applicable law or agreed to in writing, software
-//* distributed under the License is distributed on an "AS IS" BASIS,
-//* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//* See the License for the specific language governing permissions and
-//* limitations under the License.
-//*/
-//
-//package org.apache.hadoop.mapreduce.v2.app2.webapp;
-//
-//import static org.apache.hadoop.mapreduce.v2.app2.webapp.AMParams.APP_ID;
-//import static org.junit.Assert.assertEquals;
-//
-//import java.util.HashMap;
-//import java.util.Map;
-//import java.util.Map.Entry;
-//
-//import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-//import org.apache.hadoop.mapreduce.v2.app2.AppContext;
-//import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Job;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Task;
-//import org.apache.hadoop.mapreduce.v2.app2.job.TaskAttempt;
-//import org.apache.hadoop.mapreduce.v2.util.MRApps;
-//import org.apache.hadoop.yarn.Clock;
-//import org.apache.hadoop.yarn.ClusterInfo;
-//import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-//import org.apache.hadoop.yarn.api.records.ApplicationId;
-//import org.apache.hadoop.yarn.event.EventHandler;
-//import org.apache.hadoop.yarn.webapp.WebApps;
-//import org.apache.hadoop.yarn.webapp.test.WebAppTests;
-//import org.junit.Test;
-//
-//import com.google.inject.Injector;
-//
-//public class TestAMWebApp {
-//
-// static class TestAppContext implements AppContext {
-// final ApplicationAttemptId appAttemptID;
-// final ApplicationId appID;
-// final String user = MockJobs.newUserName();
-// final Map<JobId, Job> jobs;
-// final long startTime = System.currentTimeMillis();
-//
-// TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
-// appID = MockJobs.newAppID(appid);
-// appAttemptID = MockJobs.newAppAttemptID(appID, 0);
-// jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
-// }
-//
-// TestAppContext() {
-// this(0, 1, 1, 1);
-// }
-//
-// @Override
-// public ApplicationAttemptId getApplicationAttemptId() {
-// return appAttemptID;
-// }
-//
-// @Override
-// public ApplicationId getApplicationID() {
-// return appID;
-// }
-//
-// @Override
-// public CharSequence getUser() {
-// return user;
-// }
-//
-// @Override
-// public Job getJob(JobId jobID) {
-// return jobs.get(jobID);
-// }
-//
-// @Override
-// public Map<JobId, Job> getAllJobs() {
-// return jobs; // OK
-// }
-//
-// @SuppressWarnings("rawtypes")
-// @Override
-// public EventHandler getEventHandler() {
-// return null;
-// }
-//
-// @Override
-// public Clock getClock() {
-// return null;
-// }
-//
-// @Override
-// public String getApplicationName() {
-// return "TestApp";
-// }
-//
-// @Override
-// public long getStartTime() {
-// return startTime;
-// }
-//
-// @Override
-// public ClusterInfo getClusterInfo() {
-// return null;
-// }
-// }
-//
-// @Test public void testAppControllerIndex() {
-// TestAppContext ctx = new TestAppContext();
-// Injector injector = WebAppTests.createMockInjector(AppContext.class, ctx);
-// AppController controller = injector.getInstance(AppController.class);
-// controller.index();
-// assertEquals(ctx.appID.toString(), controller.get(APP_ID,""));
-// }
-//
-// @Test public void testAppView() {
-// WebAppTests.testPage(AppView.class, AppContext.class, new TestAppContext());
-// }
-//
-//
-//
-// @Test public void testJobView() {
-// AppContext appContext = new TestAppContext();
-// Map<String, String> params = getJobParams(appContext);
-// WebAppTests.testPage(JobPage.class, AppContext.class, appContext, params);
-// }
-//
-// @Test public void testTasksView() {
-// AppContext appContext = new TestAppContext();
-// Map<String, String> params = getTaskParams(appContext);
-// WebAppTests.testPage(TasksPage.class, AppContext.class, appContext, params);
-// }
-//
-// @Test public void testTaskView() {
-// AppContext appContext = new TestAppContext();
-// Map<String, String> params = getTaskParams(appContext);
-// WebAppTests.testPage(TaskPage.class, AppContext.class, appContext, params);
-// }
-//
-// public static Map<String, String> getJobParams(AppContext appContext) {
-// JobId jobId = appContext.getAllJobs().entrySet().iterator().next().getKey();
-// Map<String, String> params = new HashMap<String, String>();
-// params.put(AMParams.JOB_ID, MRApps.toString(jobId));
-// return params;
-// }
-//
-// public static Map<String, String> getTaskParams(AppContext appContext) {
-// JobId jobId = appContext.getAllJobs().entrySet().iterator().next().getKey();
-// Entry<TaskId, Task> e = appContext.getJob(jobId).getTasks().entrySet().iterator().next();
-// e.getValue().getType();
-// Map<String, String> params = new HashMap<String, String>();
-// params.put(AMParams.JOB_ID, MRApps.toString(jobId));
-// params.put(AMParams.TASK_ID, MRApps.toString(e.getKey()));
-// params.put(AMParams.TASK_TYPE, MRApps.taskSymbol(e.getValue().getType()));
-// return params;
-// }
-//
-// @Test public void testConfView() {
-// WebAppTests.testPage(JobConfPage.class, AppContext.class,
-// new TestAppContext());
-// }
-//
-// @Test public void testCountersView() {
-// AppContext appContext = new TestAppContext();
-// Map<String, String> params = getJobParams(appContext);
-// WebAppTests.testPage(CountersPage.class, AppContext.class,
-// appContext, params);
-// }
-//
-// @Test public void testSingleCounterView() {
-// AppContext appContext = new TestAppContext();
-// Map<String, String> params = getJobParams(appContext);
-// params.put(AMParams.COUNTER_GROUP,
-// "org.apache.hadoop.mapreduce.FileSystemCounter");
-// params.put(AMParams.COUNTER_NAME, "HDFS_WRITE_OPS");
-// WebAppTests.testPage(SingleCounterPage.class, AppContext.class,
-// appContext, params);
-// }
-//
-// @Test public void testTaskCountersView() {
-// AppContext appContext = new TestAppContext();
-// Map<String, String> params = getTaskParams(appContext);
-// WebAppTests.testPage(CountersPage.class, AppContext.class,
-// appContext, params);
-// }
-//
-// @Test public void testSingleTaskCounterView() {
-// AppContext appContext = new TestAppContext(0, 1, 1, 2);
-// Map<String, String> params = getTaskParams(appContext);
-// params.put(AMParams.COUNTER_GROUP,
-// "org.apache.hadoop.mapreduce.FileSystemCounter");
-// params.put(AMParams.COUNTER_NAME, "HDFS_WRITE_OPS");
-//
-// // remove counters from one task attempt
-// // to test handling of missing counters
-// TaskId taskID = MRApps.toTaskID(params.get(AMParams.TASK_ID));
-// Job job = appContext.getJob(taskID.getJobId());
-// Task task = job.getTask(taskID);
-// TaskAttempt attempt = task.getAttempts().values().iterator().next();
-// attempt.getReport().setCounters(null);
-//
-// WebAppTests.testPage(SingleCounterPage.class, AppContext.class,
-// appContext, params);
-// }
-//
-// public static void main(String[] args) {
-// WebApps.$for("yarn", AppContext.class, new TestAppContext(0, 8, 88, 4)).
-// at(58888).inDevMode().start(new AMWebApp()).joinThread();
-// }
-//}
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app2.webapp;
+
+import static org.apache.hadoop.mapreduce.v2.app2.webapp.AMParams.APP_ID;
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.app2.AppContext;
+import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
+import org.apache.hadoop.mapreduce.v2.app2.job.Job;
+import org.apache.hadoop.mapreduce.v2.app2.job.Task;
+import org.apache.hadoop.mapreduce.v2.app2.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.ClusterInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.webapp.WebApps;
+import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.junit.Test;
+
+import com.google.inject.Injector;
+
+public class TestAMWebApp {
+
+ static class TestAppContext implements AppContext {
+ final ApplicationAttemptId appAttemptID;
+ final ApplicationId appID;
+ final String user = MockJobs.newUserName();
+ final Map<JobId, Job> jobs;
+ final long startTime = System.currentTimeMillis();
+
+ TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
+ appID = MockJobs.newAppID(appid);
+ appAttemptID = MockJobs.newAppAttemptID(appID, 0);
+ jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
+ }
+
+ TestAppContext() {
+ this(0, 1, 1, 1);
+ }
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptID;
+ }
+
+ @Override
+ public ApplicationId getApplicationID() {
+ return appID;
+ }
+
+ @Override
+ public CharSequence getUser() {
+ return user;
+ }
+
+ @Override
+ public Job getJob(JobId jobID) {
+ return jobs.get(jobID);
+ }
+
+ @Override
+ public Map<JobId, Job> getAllJobs() {
+ return jobs; // OK
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public EventHandler getEventHandler() {
+ return null;
+ }
+
+ @Override
+ public Clock getClock() {
+ return null;
+ }
+
+ @Override
+ public String getApplicationName() {
+ return "TestApp";
+ }
+
+ @Override
+ public long getStartTime() {
+ return startTime;
+ }
+
+ @Override
+ public ClusterInfo getClusterInfo() {
+ return null;
+ }
+ }
+
+ @Test public void testAppControllerIndex() {
+ TestAppContext ctx = new TestAppContext();
+ Injector injector = WebAppTests.createMockInjector(AppContext.class, ctx);
+ AppController controller = injector.getInstance(AppController.class);
+ controller.index();
+ assertEquals(ctx.appID.toString(), controller.get(APP_ID,""));
+ }
+
+ @Test public void testAppView() {
+ WebAppTests.testPage(AppView.class, AppContext.class, new TestAppContext());
+ }
+
+
+
+ @Test public void testJobView() {
+ AppContext appContext = new TestAppContext();
+ Map<String, String> params = getJobParams(appContext);
+ WebAppTests.testPage(JobPage.class, AppContext.class, appContext, params);
+ }
+
+ @Test public void testTasksView() {
+ AppContext appContext = new TestAppContext();
+ Map<String, String> params = getTaskParams(appContext);
+ WebAppTests.testPage(TasksPage.class, AppContext.class, appContext, params);
+ }
+
+ @Test public void testTaskView() {
+ AppContext appContext = new TestAppContext();
+ Map<String, String> params = getTaskParams(appContext);
+ WebAppTests.testPage(TaskPage.class, AppContext.class, appContext, params);
+ }
+
+ public static Map<String, String> getJobParams(AppContext appContext) {
+ JobId jobId = appContext.getAllJobs().entrySet().iterator().next().getKey();
+ Map<String, String> params = new HashMap<String, String>();
+ params.put(AMParams.JOB_ID, MRApps.toString(jobId));
+ return params;
+ }
+
+ public static Map<String, String> getTaskParams(AppContext appContext) {
+ JobId jobId = appContext.getAllJobs().entrySet().iterator().next().getKey();
+ Entry<TaskId, Task> e = appContext.getJob(jobId).getTasks().entrySet().iterator().next();
+ e.getValue().getType();
+ Map<String, String> params = new HashMap<String, String>();
+ params.put(AMParams.JOB_ID, MRApps.toString(jobId));
+ params.put(AMParams.TASK_ID, MRApps.toString(e.getKey()));
+ params.put(AMParams.TASK_TYPE, MRApps.taskSymbol(e.getValue().getType()));
+ return params;
+ }
+
+ @Test public void testConfView() {
+ WebAppTests.testPage(JobConfPage.class, AppContext.class,
+ new TestAppContext());
+ }
+
+ @Test public void testCountersView() {
+ AppContext appContext = new TestAppContext();
+ Map<String, String> params = getJobParams(appContext);
+ WebAppTests.testPage(CountersPage.class, AppContext.class,
+ appContext, params);
+ }
+
+ @Test public void testSingleCounterView() {
+ AppContext appContext = new TestAppContext();
+ Map<String, String> params = getJobParams(appContext);
+ params.put(AMParams.COUNTER_GROUP,
+ "org.apache.hadoop.mapreduce.FileSystemCounter");
+ params.put(AMParams.COUNTER_NAME, "HDFS_WRITE_OPS");
+ WebAppTests.testPage(SingleCounterPage.class, AppContext.class,
+ appContext, params);
+ }
+
+ @Test public void testTaskCountersView() {
+ AppContext appContext = new TestAppContext();
+ Map<String, String> params = getTaskParams(appContext);
+ WebAppTests.testPage(CountersPage.class, AppContext.class,
+ appContext, params);
+ }
+
+ @Test public void testSingleTaskCounterView() {
+ AppContext appContext = new TestAppContext(0, 1, 1, 2);
+ Map<String, String> params = getTaskParams(appContext);
+ params.put(AMParams.COUNTER_GROUP,
+ "org.apache.hadoop.mapreduce.FileSystemCounter");
+ params.put(AMParams.COUNTER_NAME, "HDFS_WRITE_OPS");
+
+ // remove counters from one task attempt
+ // to test handling of missing counters
+ TaskId taskID = MRApps.toTaskID(params.get(AMParams.TASK_ID));
+ Job job = appContext.getJob(taskID.getJobId());
+ Task task = job.getTask(taskID);
+ TaskAttempt attempt = task.getAttempts().values().iterator().next();
+ attempt.getReport().setCounters(null);
+
+ WebAppTests.testPage(SingleCounterPage.class, AppContext.class,
+ appContext, params);
+ }
+
+ public static void main(String[] args) {
+ WebApps.$for("yarn", AppContext.class, new TestAppContext(0, 8, 88, 4)).
+ at(58888).inDevMode().start(new AMWebApp()).joinThread();
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServices.java
index f4da3f8..d60340a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServices.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServices.java
@@ -1,365 +1,365 @@
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.hadoop.mapreduce.v2.app2.webapp;
-//
-//import static org.junit.Assert.assertEquals;
-//import static org.junit.Assert.assertTrue;
-//import static org.junit.Assert.fail;
-//
-//import java.io.StringReader;
-//import java.util.Map;
-//
-//import javax.ws.rs.core.MediaType;
-//import javax.xml.parsers.DocumentBuilder;
-//import javax.xml.parsers.DocumentBuilderFactory;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-//import org.apache.hadoop.mapreduce.v2.app2.AppContext;
-//import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Job;
-//import org.apache.hadoop.yarn.Clock;
-//import org.apache.hadoop.yarn.ClusterInfo;
-//import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-//import org.apache.hadoop.yarn.api.records.ApplicationId;
-//import org.apache.hadoop.yarn.event.EventHandler;
-//import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-//import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
-//import org.codehaus.jettison.json.JSONException;
-//import org.codehaus.jettison.json.JSONObject;
-//import org.junit.Before;
-//import org.junit.Test;
-//import org.w3c.dom.Document;
-//import org.w3c.dom.Element;
-//import org.w3c.dom.NodeList;
-//import org.xml.sax.InputSource;
-//
-//import com.google.inject.Guice;
-//import com.google.inject.Injector;
-//import com.google.inject.servlet.GuiceServletContextListener;
-//import com.google.inject.servlet.ServletModule;
-//import com.sun.jersey.api.client.ClientResponse;
-//import com.sun.jersey.api.client.ClientResponse.Status;
-//import com.sun.jersey.api.client.UniformInterfaceException;
-//import com.sun.jersey.api.client.WebResource;
-//import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-//import com.sun.jersey.test.framework.JerseyTest;
-//import com.sun.jersey.test.framework.WebAppDescriptor;
-//
-///**
-// * Test the MapReduce Application master info web services api's. Also test
-// * non-existent urls.
-// *
-// * /ws/v1/mapreduce
-// * /ws/v1/mapreduce/info
-// */
-//public class TestAMWebServices extends JerseyTest {
-//
-// private static Configuration conf = new Configuration();
-// private static TestAppContext appContext;
-//
-// static class TestAppContext implements AppContext {
-// final ApplicationAttemptId appAttemptID;
-// final ApplicationId appID;
-// final String user = MockJobs.newUserName();
-// final Map<JobId, Job> jobs;
-// final long startTime = System.currentTimeMillis();
-//
-// TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
-// appID = MockJobs.newAppID(appid);
-// appAttemptID = MockJobs.newAppAttemptID(appID, 0);
-// jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
-// }
-//
-// TestAppContext() {
-// this(0, 1, 1, 1);
-// }
-//
-// @Override
-// public ApplicationAttemptId getApplicationAttemptId() {
-// return appAttemptID;
-// }
-//
-// @Override
-// public ApplicationId getApplicationID() {
-// return appID;
-// }
-//
-// @Override
-// public CharSequence getUser() {
-// return user;
-// }
-//
-// @Override
-// public Job getJob(JobId jobID) {
-// return jobs.get(jobID);
-// }
-//
-// @Override
-// public Map<JobId, Job> getAllJobs() {
-// return jobs; // OK
-// }
-//
-// @SuppressWarnings("rawtypes")
-// @Override
-// public EventHandler getEventHandler() {
-// return null;
-// }
-//
-// @Override
-// public Clock getClock() {
-// return null;
-// }
-//
-// @Override
-// public String getApplicationName() {
-// return "TestApp";
-// }
-//
-// @Override
-// public long getStartTime() {
-// return startTime;
-// }
-//
-// @Override
-// public ClusterInfo getClusterInfo() {
-// return null;
-// }
-// }
-//
-// private Injector injector = Guice.createInjector(new ServletModule() {
-// @Override
-// protected void configureServlets() {
-//
-// appContext = new TestAppContext();
-// bind(JAXBContextResolver.class);
-// bind(AMWebServices.class);
-// bind(GenericExceptionHandler.class);
-// bind(AppContext.class).toInstance(appContext);
-// bind(Configuration.class).toInstance(conf);
-//
-// serve("/*").with(GuiceContainer.class);
-// }
-// });
-//
-// public class GuiceServletConfig extends GuiceServletContextListener {
-//
-// @Override
-// protected Injector getInjector() {
-// return injector;
-// }
-// }
-//
-// @Before
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-// }
-//
-// public TestAMWebServices() {
-// super(new WebAppDescriptor.Builder(
-// "org.apache.hadoop.mapreduce.v2.app2.webapp")
-// .contextListenerClass(GuiceServletConfig.class)
-// .filterClass(com.google.inject.servlet.GuiceFilter.class)
-// .contextPath("jersey-guice-filter").servletPath("/").build());
-// }
-//
-// @Test
-// public void testAM() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// verifyAMInfo(json.getJSONObject("info"), appContext);
-// }
-//
-// @Test
-// public void testAMSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce/")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// verifyAMInfo(json.getJSONObject("info"), appContext);
-// }
-//
-// @Test
-// public void testAMDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce/")
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// verifyAMInfo(json.getJSONObject("info"), appContext);
-// }
-//
-// @Test
-// public void testAMXML() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// verifyAMInfoXML(xml, appContext);
-// }
-//
-// @Test
-// public void testInfo() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("info").accept(MediaType.APPLICATION_JSON)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// verifyAMInfo(json.getJSONObject("info"), appContext);
-// }
-//
-// @Test
-// public void testInfoSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("info/").accept(MediaType.APPLICATION_JSON)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// verifyAMInfo(json.getJSONObject("info"), appContext);
-// }
-//
-// @Test
-// public void testInfoDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("info/").get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// verifyAMInfo(json.getJSONObject("info"), appContext);
-// }
-//
-// @Test
-// public void testInfoXML() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("info/").accept(MediaType.APPLICATION_XML)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// verifyAMInfoXML(xml, appContext);
-// }
-//
-// @Test
-// public void testInvalidUri() throws JSONException, Exception {
-// WebResource r = resource();
-// String responseStr = "";
-// try {
-// responseStr = r.path("ws").path("v1").path("mapreduce").path("bogus")
-// .accept(MediaType.APPLICATION_JSON).get(String.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// WebServicesTestUtils.checkStringMatch(
-// "error string exists and shouldn't", "", responseStr);
-// }
-// }
-//
-// @Test
-// public void testInvalidUri2() throws JSONException, Exception {
-// WebResource r = resource();
-// String responseStr = "";
-// try {
-// responseStr = r.path("ws").path("v1").path("invalid")
-// .accept(MediaType.APPLICATION_JSON).get(String.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// WebServicesTestUtils.checkStringMatch(
-// "error string exists and shouldn't", "", responseStr);
-// }
-// }
-//
-// @Test
-// public void testInvalidAccept() throws JSONException, Exception {
-// WebResource r = resource();
-// String responseStr = "";
-// try {
-// responseStr = r.path("ws").path("v1").path("mapreduce")
-// .accept(MediaType.TEXT_PLAIN).get(String.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.INTERNAL_SERVER_ERROR,
-// response.getClientResponseStatus());
-// WebServicesTestUtils.checkStringMatch(
-// "error string exists and shouldn't", "", responseStr);
-// }
-// }
-//
-// public void verifyAMInfo(JSONObject info, TestAppContext ctx)
-// throws JSONException {
-// assertEquals("incorrect number of elements", 5, info.length());
-//
-// verifyAMInfoGeneric(ctx, info.getString("appId"), info.getString("user"),
-// info.getString("name"), info.getLong("startedOn"),
-// info.getLong("elapsedTime"));
-// }
-//
-// public void verifyAMInfoXML(String xml, TestAppContext ctx)
-// throws JSONException, Exception {
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList nodes = dom.getElementsByTagName("info");
-// assertEquals("incorrect number of elements", 1, nodes.getLength());
-//
-// for (int i = 0; i < nodes.getLength(); i++) {
-// Element element = (Element) nodes.item(i);
-// verifyAMInfoGeneric(ctx,
-// WebServicesTestUtils.getXmlString(element, "appId"),
-// WebServicesTestUtils.getXmlString(element, "user"),
-// WebServicesTestUtils.getXmlString(element, "name"),
-// WebServicesTestUtils.getXmlLong(element, "startedOn"),
-// WebServicesTestUtils.getXmlLong(element, "elapsedTime"));
-// }
-// }
-//
-// public void verifyAMInfoGeneric(TestAppContext ctx, String id, String user,
-// String name, long startedOn, long elapsedTime) {
-//
-// WebServicesTestUtils.checkStringMatch("id", ctx.getApplicationID()
-// .toString(), id);
-// WebServicesTestUtils.checkStringMatch("user", ctx.getUser().toString(),
-// user);
-// WebServicesTestUtils.checkStringMatch("name", ctx.getApplicationName(),
-// name);
-//
-// assertEquals("startedOn incorrect", ctx.getStartTime(), startedOn);
-// assertTrue("elapsedTime not greater then 0", (elapsedTime > 0));
-//
-// }
-//}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.app2.webapp;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.StringReader;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.app2.AppContext;
+import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
+import org.apache.hadoop.mapreduce.v2.app2.job.Job;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.ClusterInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+/**
+ * Test the MapReduce Application master info web services api's. Also test
+ * non-existent urls.
+ *
+ * /ws/v1/mapreduce
+ * /ws/v1/mapreduce/info
+ */
+public class TestAMWebServices extends JerseyTest {
+
+ private static Configuration conf = new Configuration();
+ private static TestAppContext appContext;
+
+ static class TestAppContext implements AppContext {
+ final ApplicationAttemptId appAttemptID;
+ final ApplicationId appID;
+ final String user = MockJobs.newUserName();
+ final Map<JobId, Job> jobs;
+ final long startTime = System.currentTimeMillis();
+
+ TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
+ appID = MockJobs.newAppID(appid);
+ appAttemptID = MockJobs.newAppAttemptID(appID, 0);
+ jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
+ }
+
+ TestAppContext() {
+ this(0, 1, 1, 1);
+ }
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptID;
+ }
+
+ @Override
+ public ApplicationId getApplicationID() {
+ return appID;
+ }
+
+ @Override
+ public CharSequence getUser() {
+ return user;
+ }
+
+ @Override
+ public Job getJob(JobId jobID) {
+ return jobs.get(jobID);
+ }
+
+ @Override
+ public Map<JobId, Job> getAllJobs() {
+ return jobs; // OK
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public EventHandler getEventHandler() {
+ return null;
+ }
+
+ @Override
+ public Clock getClock() {
+ return null;
+ }
+
+ @Override
+ public String getApplicationName() {
+ return "TestApp";
+ }
+
+ @Override
+ public long getStartTime() {
+ return startTime;
+ }
+
+ @Override
+ public ClusterInfo getClusterInfo() {
+ return null;
+ }
+ }
+
+ private Injector injector = Guice.createInjector(new ServletModule() {
+ @Override
+ protected void configureServlets() {
+
+ appContext = new TestAppContext();
+ bind(JAXBContextResolver.class);
+ bind(AMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ bind(AppContext.class).toInstance(appContext);
+ bind(Configuration.class).toInstance(conf);
+
+ serve("/*").with(GuiceContainer.class);
+ }
+ });
+
+ public class GuiceServletConfig extends GuiceServletContextListener {
+
+ @Override
+ protected Injector getInjector() {
+ return injector;
+ }
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ public TestAMWebServices() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.mapreduce.v2.app2.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testAM() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ verifyAMInfo(json.getJSONObject("info"), appContext);
+ }
+
+ @Test
+ public void testAMSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ verifyAMInfo(json.getJSONObject("info"), appContext);
+ }
+
+ @Test
+ public void testAMDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce/")
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ verifyAMInfo(json.getJSONObject("info"), appContext);
+ }
+
+ @Test
+ public void testAMXML() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ verifyAMInfoXML(xml, appContext);
+ }
+
+ @Test
+ public void testInfo() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("info").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ verifyAMInfo(json.getJSONObject("info"), appContext);
+ }
+
+ @Test
+ public void testInfoSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("info/").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ verifyAMInfo(json.getJSONObject("info"), appContext);
+ }
+
+ @Test
+ public void testInfoDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("info/").get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ verifyAMInfo(json.getJSONObject("info"), appContext);
+ }
+
+ @Test
+ public void testInfoXML() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("info/").accept(MediaType.APPLICATION_XML)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ verifyAMInfoXML(xml, appContext);
+ }
+
+ @Test
+ public void testInvalidUri() throws JSONException, Exception {
+ WebResource r = resource();
+ String responseStr = "";
+ try {
+ responseStr = r.path("ws").path("v1").path("mapreduce").path("bogus")
+ .accept(MediaType.APPLICATION_JSON).get(String.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ WebServicesTestUtils.checkStringMatch(
+ "error string exists and shouldn't", "", responseStr);
+ }
+ }
+
+ @Test
+ public void testInvalidUri2() throws JSONException, Exception {
+ WebResource r = resource();
+ String responseStr = "";
+ try {
+ responseStr = r.path("ws").path("v1").path("invalid")
+ .accept(MediaType.APPLICATION_JSON).get(String.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ WebServicesTestUtils.checkStringMatch(
+ "error string exists and shouldn't", "", responseStr);
+ }
+ }
+
+ @Test
+ public void testInvalidAccept() throws JSONException, Exception {
+ WebResource r = resource();
+ String responseStr = "";
+ try {
+ responseStr = r.path("ws").path("v1").path("mapreduce")
+ .accept(MediaType.TEXT_PLAIN).get(String.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.INTERNAL_SERVER_ERROR,
+ response.getClientResponseStatus());
+ WebServicesTestUtils.checkStringMatch(
+ "error string exists and shouldn't", "", responseStr);
+ }
+ }
+
+ public void verifyAMInfo(JSONObject info, TestAppContext ctx)
+ throws JSONException {
+ assertEquals("incorrect number of elements", 5, info.length());
+
+ verifyAMInfoGeneric(ctx, info.getString("appId"), info.getString("user"),
+ info.getString("name"), info.getLong("startedOn"),
+ info.getLong("elapsedTime"));
+ }
+
+ public void verifyAMInfoXML(String xml, TestAppContext ctx)
+ throws JSONException, Exception {
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList nodes = dom.getElementsByTagName("info");
+ assertEquals("incorrect number of elements", 1, nodes.getLength());
+
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+ verifyAMInfoGeneric(ctx,
+ WebServicesTestUtils.getXmlString(element, "appId"),
+ WebServicesTestUtils.getXmlString(element, "user"),
+ WebServicesTestUtils.getXmlString(element, "name"),
+ WebServicesTestUtils.getXmlLong(element, "startedOn"),
+ WebServicesTestUtils.getXmlLong(element, "elapsedTime"));
+ }
+ }
+
+ public void verifyAMInfoGeneric(TestAppContext ctx, String id, String user,
+ String name, long startedOn, long elapsedTime) {
+
+ WebServicesTestUtils.checkStringMatch("id", ctx.getApplicationID()
+ .toString(), id);
+ WebServicesTestUtils.checkStringMatch("user", ctx.getUser().toString(),
+ user);
+ WebServicesTestUtils.checkStringMatch("name", ctx.getApplicationName(),
+ name);
+
+ assertEquals("startedOn incorrect", ctx.getStartTime(), startedOn);
+ assertTrue("elapsedTime not greater then 0", (elapsedTime > 0));
+
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesAttempts.java
index 4fac5e4..8492497 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesAttempts.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesAttempts.java
@@ -1,738 +1,738 @@
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.hadoop.mapreduce.v2.app2.webapp;
-//
-//import static org.junit.Assert.assertEquals;
-//import static org.junit.Assert.assertNotNull;
-//import static org.junit.Assert.assertTrue;
-//import static org.junit.Assert.fail;
-//
-//import java.io.StringReader;
-//import java.util.List;
-//import java.util.Map;
-//
-//import javax.ws.rs.core.MediaType;
-//import javax.xml.parsers.DocumentBuilder;
-//import javax.xml.parsers.DocumentBuilderFactory;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-//import org.apache.hadoop.mapreduce.v2.app2.AppContext;
-//import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Job;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Task;
-//import org.apache.hadoop.mapreduce.v2.app2.job.TaskAttempt;
-//import org.apache.hadoop.mapreduce.v2.util.MRApps;
-//import org.apache.hadoop.yarn.Clock;
-//import org.apache.hadoop.yarn.ClusterInfo;
-//import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-//import org.apache.hadoop.yarn.api.records.ApplicationId;
-//import org.apache.hadoop.yarn.event.EventHandler;
-//import org.apache.hadoop.yarn.util.ConverterUtils;
-//import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-//import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
-//import org.codehaus.jettison.json.JSONArray;
-//import org.codehaus.jettison.json.JSONException;
-//import org.codehaus.jettison.json.JSONObject;
-//import org.junit.Before;
-//import org.junit.Test;
-//import org.w3c.dom.Document;
-//import org.w3c.dom.Element;
-//import org.w3c.dom.NodeList;
-//import org.xml.sax.InputSource;
-//
-//import com.google.inject.Guice;
-//import com.google.inject.Injector;
-//import com.google.inject.servlet.GuiceServletContextListener;
-//import com.google.inject.servlet.ServletModule;
-//import com.sun.jersey.api.client.ClientResponse;
-//import com.sun.jersey.api.client.UniformInterfaceException;
-//import com.sun.jersey.api.client.WebResource;
-//import com.sun.jersey.api.client.ClientResponse.Status;
-//import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-//import com.sun.jersey.test.framework.JerseyTest;
-//import com.sun.jersey.test.framework.WebAppDescriptor;
-//
-///**
-// * Test the app master web service Rest API for getting task attempts, a
-// * specific task attempt, and task attempt counters
-// *
-// * /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts
-// * /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}
-// * /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}/counters
-// */
-//public class TestAMWebServicesAttempts extends JerseyTest {
-//
-// private static Configuration conf = new Configuration();
-// private static TestAppContext appContext;
-//
-// static class TestAppContext implements AppContext {
-// final ApplicationAttemptId appAttemptID;
-// final ApplicationId appID;
-// final String user = MockJobs.newUserName();
-// final Map<JobId, Job> jobs;
-// final long startTime = System.currentTimeMillis();
-//
-// TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
-// appID = MockJobs.newAppID(appid);
-// appAttemptID = MockJobs.newAppAttemptID(appID, 0);
-// jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
-// }
-//
-// TestAppContext() {
-// this(0, 1, 2, 1);
-// }
-//
-// @Override
-// public ApplicationAttemptId getApplicationAttemptId() {
-// return appAttemptID;
-// }
-//
-// @Override
-// public ApplicationId getApplicationID() {
-// return appID;
-// }
-//
-// @Override
-// public CharSequence getUser() {
-// return user;
-// }
-//
-// @Override
-// public Job getJob(JobId jobID) {
-// return jobs.get(jobID);
-// }
-//
-// @Override
-// public Map<JobId, Job> getAllJobs() {
-// return jobs; // OK
-// }
-//
-// @SuppressWarnings("rawtypes")
-// @Override
-// public EventHandler getEventHandler() {
-// return null;
-// }
-//
-// @Override
-// public Clock getClock() {
-// return null;
-// }
-//
-// @Override
-// public String getApplicationName() {
-// return "TestApp";
-// }
-//
-// @Override
-// public long getStartTime() {
-// return startTime;
-// }
-//
-// @Override
-// public ClusterInfo getClusterInfo() {
-// return null;
-// }
-// }
-//
-// private Injector injector = Guice.createInjector(new ServletModule() {
-// @Override
-// protected void configureServlets() {
-//
-// appContext = new TestAppContext();
-// bind(JAXBContextResolver.class);
-// bind(AMWebServices.class);
-// bind(GenericExceptionHandler.class);
-// bind(AppContext.class).toInstance(appContext);
-// bind(Configuration.class).toInstance(conf);
-//
-// serve("/*").with(GuiceContainer.class);
-// }
-// });
-//
-// public class GuiceServletConfig extends GuiceServletContextListener {
-//
-// @Override
-// protected Injector getInjector() {
-// return injector;
-// }
-// }
-//
-// @Before
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// }
-//
-// public TestAMWebServicesAttempts() {
-// super(new WebAppDescriptor.Builder(
-// "org.apache.hadoop.mapreduce.v2.app2.webapp")
-// .contextListenerClass(GuiceServletConfig.class)
-// .filterClass(com.google.inject.servlet.GuiceFilter.class)
-// .contextPath("jersey-guice-filter").servletPath("/").build());
-// }
-//
-// @Test
-// public void testTaskAttempts() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid).path("attempts")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// verifyAMTaskAttempts(json, task);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskAttemptsSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid).path("attempts/")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// verifyAMTaskAttempts(json, task);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskAttemptsDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid).path("attempts")
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// verifyAMTaskAttempts(json, task);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskAttemptsXML() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid).path("attempts")
-// .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-//
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList attempts = dom.getElementsByTagName("taskAttempts");
-// assertEquals("incorrect number of elements", 1, attempts.getLength());
-//
-// NodeList nodes = dom.getElementsByTagName("taskAttempt");
-// verifyAMTaskAttemptsXML(nodes, task);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskAttemptId() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-//
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-// String tid = MRApps.toString(task.getID());
-//
-// for (TaskAttempt att : task.getAttempts().values()) {
-// TaskAttemptId attemptid = att.getID();
-// String attid = MRApps.toString(attemptid);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid)
-// .path("attempts").path(attid).accept(MediaType.APPLICATION_JSON)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("taskAttempt");
-// verifyAMTaskAttempt(info, att, task.getType());
-// }
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskAttemptIdSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-//
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-// String tid = MRApps.toString(task.getID());
-//
-// for (TaskAttempt att : task.getAttempts().values()) {
-// TaskAttemptId attemptid = att.getID();
-// String attid = MRApps.toString(attemptid);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid)
-// .path("attempts").path(attid + "/")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("taskAttempt");
-// verifyAMTaskAttempt(info, att, task.getType());
-// }
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskAttemptIdDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-//
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-// String tid = MRApps.toString(task.getID());
-//
-// for (TaskAttempt att : task.getAttempts().values()) {
-// TaskAttemptId attemptid = att.getID();
-// String attid = MRApps.toString(attemptid);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid)
-// .path("attempts").path(attid).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("taskAttempt");
-// verifyAMTaskAttempt(info, att, task.getType());
-// }
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskAttemptIdXML() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// for (TaskAttempt att : task.getAttempts().values()) {
-// TaskAttemptId attemptid = att.getID();
-// String attid = MRApps.toString(attemptid);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid)
-// .path("attempts").path(attid).accept(MediaType.APPLICATION_XML)
-// .get(ClientResponse.class);
-//
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList nodes = dom.getElementsByTagName("taskAttempt");
-// for (int i = 0; i < nodes.getLength(); i++) {
-// Element element = (Element) nodes.item(i);
-// verifyAMTaskAttemptXML(element, att, task.getType());
-// }
-// }
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskAttemptIdBogus() throws JSONException, Exception {
-//
-// testTaskAttemptIdErrorGeneric("bogusid",
-// "java.lang.Exception: TaskAttemptId string : bogusid is not properly formed");
-// }
-//
-// @Test
-// public void testTaskAttemptIdNonExist() throws JSONException, Exception {
-//
-// testTaskAttemptIdErrorGeneric(
-// "attempt_0_12345_m_000000_0",
-// "java.lang.Exception: Error getting info on task attempt id attempt_0_12345_m_000000_0");
-// }
-//
-// @Test
-// public void testTaskAttemptIdInvalid() throws JSONException, Exception {
-//
-// testTaskAttemptIdErrorGeneric("attempt_0_12345_d_000000_0",
-// "java.lang.Exception: Bad TaskType identifier. TaskAttemptId string : attempt_0_12345_d_000000_0 is not properly formed.");
-// }
-//
-// @Test
-// public void testTaskAttemptIdInvalid2() throws JSONException, Exception {
-//
-// testTaskAttemptIdErrorGeneric("attempt_12345_m_000000_0",
-// "java.lang.Exception: TaskAttemptId string : attempt_12345_m_000000_0 is not properly formed");
-// }
-//
-// @Test
-// public void testTaskAttemptIdInvalid3() throws JSONException, Exception {
-//
-// testTaskAttemptIdErrorGeneric("attempt_0_12345_m_000000",
-// "java.lang.Exception: TaskAttemptId string : attempt_0_12345_m_000000 is not properly formed");
-// }
-//
-// private void testTaskAttemptIdErrorGeneric(String attid, String error)
-// throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-//
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-// String tid = MRApps.toString(task.getID());
-//
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
-// .path("tasks").path(tid).path("attempts").path(attid)
-// .accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// WebServicesTestUtils.checkStringMatch("exception message", error,
-// message);
-// WebServicesTestUtils.checkStringMatch("exception type",
-// "NotFoundException", type);
-// WebServicesTestUtils.checkStringMatch("exception classname",
-// "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
-// }
-// }
-// }
-// }
-//
-// public void verifyAMTaskAttemptXML(Element element, TaskAttempt att,
-// TaskType ttype) {
-// verifyTaskAttemptGeneric(att, ttype,
-// WebServicesTestUtils.getXmlString(element, "id"),
-// WebServicesTestUtils.getXmlString(element, "state"),
-// WebServicesTestUtils.getXmlString(element, "type"),
-// WebServicesTestUtils.getXmlString(element, "rack"),
-// WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
-// WebServicesTestUtils.getXmlString(element, "diagnostics"),
-// WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
-// WebServicesTestUtils.getXmlLong(element, "startTime"),
-// WebServicesTestUtils.getXmlLong(element, "finishTime"),
-// WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
-// WebServicesTestUtils.getXmlFloat(element, "progress"));
-//
-// if (ttype == TaskType.REDUCE) {
-// verifyReduceTaskAttemptGeneric(att,
-// WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
-// WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
-// WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
-// WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
-// WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
-// }
-// }
-//
-// public void verifyAMTaskAttempt(JSONObject info, TaskAttempt att,
-// TaskType ttype) throws JSONException {
-// if (ttype == TaskType.REDUCE) {
-// assertEquals("incorrect number of elements", 16, info.length());
-// } else {
-// assertEquals("incorrect number of elements", 11, info.length());
-// }
-//
-// verifyTaskAttemptGeneric(att, ttype, info.getString("id"),
-// info.getString("state"), info.getString("type"),
-// info.getString("rack"), info.getString("nodeHttpAddress"),
-// info.getString("diagnostics"), info.getString("assignedContainerId"),
-// info.getLong("startTime"), info.getLong("finishTime"),
-// info.getLong("elapsedTime"), (float) info.getDouble("progress"));
-//
-// if (ttype == TaskType.REDUCE) {
-// verifyReduceTaskAttemptGeneric(att, info.getLong("shuffleFinishTime"),
-// info.getLong("mergeFinishTime"), info.getLong("elapsedShuffleTime"),
-// info.getLong("elapsedMergeTime"), info.getLong("elapsedReduceTime"));
-// }
-// }
-//
-// public void verifyAMTaskAttempts(JSONObject json, Task task)
-// throws JSONException {
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject attempts = json.getJSONObject("taskAttempts");
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONArray arr = attempts.getJSONArray("taskAttempt");
-// for (TaskAttempt att : task.getAttempts().values()) {
-// TaskAttemptId id = att.getID();
-// String attid = MRApps.toString(id);
-// Boolean found = false;
-//
-// for (int i = 0; i < arr.length(); i++) {
-// JSONObject info = arr.getJSONObject(i);
-// if (attid.matches(info.getString("id"))) {
-// found = true;
-// verifyAMTaskAttempt(info, att, task.getType());
-// }
-// }
-// assertTrue("task attempt with id: " + attid
-// + " not in web service output", found);
-// }
-// }
-//
-// public void verifyAMTaskAttemptsXML(NodeList nodes, Task task) {
-// assertEquals("incorrect number of elements", 1, nodes.getLength());
-//
-// for (TaskAttempt att : task.getAttempts().values()) {
-// TaskAttemptId id = att.getID();
-// String attid = MRApps.toString(id);
-// Boolean found = false;
-// for (int i = 0; i < nodes.getLength(); i++) {
-// Element element = (Element) nodes.item(i);
-//
-// if (attid.matches(WebServicesTestUtils.getXmlString(element, "id"))) {
-// found = true;
-// verifyAMTaskAttemptXML(element, att, task.getType());
-// }
-// }
-// assertTrue("task with id: " + attid + " not in web service output", found);
-// }
-// }
-//
-// public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype,
-// String id, String state, String type, String rack,
-// String nodeHttpAddress, String diagnostics, String assignedContainerId,
-// long startTime, long finishTime, long elapsedTime, float progress) {
-//
-// TaskAttemptId attid = ta.getID();
-// String attemptId = MRApps.toString(attid);
-//
-// WebServicesTestUtils.checkStringMatch("id", attemptId, id);
-// WebServicesTestUtils.checkStringMatch("type", ttype.toString(), type);
-// WebServicesTestUtils.checkStringMatch("state", ta.getState().toString(),
-// state);
-// WebServicesTestUtils.checkStringMatch("rack", ta.getNodeRackName(), rack);
-// WebServicesTestUtils.checkStringMatch("nodeHttpAddress",
-// ta.getNodeHttpAddress(), nodeHttpAddress);
-//
-// String expectDiag = "";
-// List<String> diagnosticsList = ta.getDiagnostics();
-// if (diagnosticsList != null && !diagnostics.isEmpty()) {
-// StringBuffer b = new StringBuffer();
-// for (String diag : diagnosticsList) {
-// b.append(diag);
-// }
-// expectDiag = b.toString();
-// }
-// WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
-// diagnostics);
-// WebServicesTestUtils.checkStringMatch("assignedContainerId",
-// ConverterUtils.toString(ta.getAssignedContainerID()),
-// assignedContainerId);
-//
-// assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
-// assertEquals("finishTime wrong", ta.getFinishTime(), finishTime);
-// assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
-// assertEquals("progress wrong", ta.getProgress() * 100, progress, 1e-3f);
-// }
-//
-// public void verifyReduceTaskAttemptGeneric(TaskAttempt ta,
-// long shuffleFinishTime, long mergeFinishTime, long elapsedShuffleTime,
-// long elapsedMergeTime, long elapsedReduceTime) {
-//
-// assertEquals("shuffleFinishTime wrong", ta.getShuffleFinishTime(),
-// shuffleFinishTime);
-// assertEquals("mergeFinishTime wrong", ta.getSortFinishTime(),
-// mergeFinishTime);
-// assertEquals("elapsedShuffleTime wrong",
-// ta.getLaunchTime() - ta.getShuffleFinishTime(), elapsedShuffleTime);
-// assertEquals("elapsedMergeTime wrong",
-// ta.getShuffleFinishTime() - ta.getSortFinishTime(), elapsedMergeTime);
-// assertEquals("elapsedReduceTime wrong",
-// ta.getSortFinishTime() - ta.getFinishTime(), elapsedReduceTime);
-// }
-//
-// @Test
-// public void testTaskAttemptIdCounters() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-//
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-// String tid = MRApps.toString(task.getID());
-//
-// for (TaskAttempt att : task.getAttempts().values()) {
-// TaskAttemptId attemptid = att.getID();
-// String attid = MRApps.toString(attemptid);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid)
-// .path("attempts").path(attid).path("counters")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("jobTaskAttemptCounters");
-// verifyAMJobTaskAttemptCounters(info, att);
-// }
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskAttemptIdXMLCounters() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// for (TaskAttempt att : task.getAttempts().values()) {
-// TaskAttemptId attemptid = att.getID();
-// String attid = MRApps.toString(attemptid);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid)
-// .path("attempts").path(attid).path("counters")
-// .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-//
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList nodes = dom.getElementsByTagName("jobTaskAttemptCounters");
-//
-// verifyAMTaskCountersXML(nodes, att);
-// }
-// }
-// }
-// }
-//
-// public void verifyAMJobTaskAttemptCounters(JSONObject info, TaskAttempt att)
-// throws JSONException {
-//
-// assertEquals("incorrect number of elements", 2, info.length());
-//
-// WebServicesTestUtils.checkStringMatch("id", MRApps.toString(att.getID()),
-// info.getString("id"));
-//
-// // just do simple verification of fields - not data is correct
-// // in the fields
-// JSONArray counterGroups = info.getJSONArray("taskAttemptCounterGroup");
-// for (int i = 0; i < counterGroups.length(); i++) {
-// JSONObject counterGroup = counterGroups.getJSONObject(i);
-// String name = counterGroup.getString("counterGroupName");
-// assertTrue("name not set", (name != null && !name.isEmpty()));
-// JSONArray counters = counterGroup.getJSONArray("counter");
-// for (int j = 0; j < counters.length(); j++) {
-// JSONObject counter = counters.getJSONObject(j);
-// String counterName = counter.getString("name");
-// assertTrue("name not set",
-// (counterName != null && !counterName.isEmpty()));
-// long value = counter.getLong("value");
-// assertTrue("value >= 0", value >= 0);
-// }
-// }
-// }
-//
-// public void verifyAMTaskCountersXML(NodeList nodes, TaskAttempt att) {
-//
-// for (int i = 0; i < nodes.getLength(); i++) {
-//
-// Element element = (Element) nodes.item(i);
-// WebServicesTestUtils.checkStringMatch("id", MRApps.toString(att.getID()),
-// WebServicesTestUtils.getXmlString(element, "id"));
-// // just do simple verification of fields - not data is correct
-// // in the fields
-// NodeList groups = element.getElementsByTagName("taskAttemptCounterGroup");
-//
-// for (int j = 0; j < groups.getLength(); j++) {
-// Element counters = (Element) groups.item(j);
-// assertNotNull("should have counters in the web service info", counters);
-// String name = WebServicesTestUtils.getXmlString(counters,
-// "counterGroupName");
-// assertTrue("name not set", (name != null && !name.isEmpty()));
-// NodeList counterArr = counters.getElementsByTagName("counter");
-// for (int z = 0; z < counterArr.getLength(); z++) {
-// Element counter = (Element) counterArr.item(z);
-// String counterName = WebServicesTestUtils.getXmlString(counter,
-// "name");
-// assertTrue("counter name not set",
-// (counterName != null && !counterName.isEmpty()));
-//
-// long value = WebServicesTestUtils.getXmlLong(counter, "value");
-// assertTrue("value not >= 0", value >= 0);
-//
-// }
-// }
-// }
-// }
-//
-//}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.app2.webapp;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.StringReader;
+import java.util.List;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app2.AppContext;
+import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
+import org.apache.hadoop.mapreduce.v2.app2.job.Job;
+import org.apache.hadoop.mapreduce.v2.app2.job.Task;
+import org.apache.hadoop.mapreduce.v2.app2.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.ClusterInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+/**
+ * Test the app master web service Rest API for getting task attempts, a
+ * specific task attempt, and task attempt counters
+ *
+ * /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts
+ * /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}
+ * /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}/counters
+ */
+public class TestAMWebServicesAttempts extends JerseyTest {
+
+ private static Configuration conf = new Configuration();
+ private static TestAppContext appContext;
+
+ static class TestAppContext implements AppContext {
+ final ApplicationAttemptId appAttemptID;
+ final ApplicationId appID;
+ final String user = MockJobs.newUserName();
+ final Map<JobId, Job> jobs;
+ final long startTime = System.currentTimeMillis();
+
+ TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
+ appID = MockJobs.newAppID(appid);
+ appAttemptID = MockJobs.newAppAttemptID(appID, 0);
+ jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
+ }
+
+ TestAppContext() {
+ this(0, 1, 2, 1);
+ }
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptID;
+ }
+
+ @Override
+ public ApplicationId getApplicationID() {
+ return appID;
+ }
+
+ @Override
+ public CharSequence getUser() {
+ return user;
+ }
+
+ @Override
+ public Job getJob(JobId jobID) {
+ return jobs.get(jobID);
+ }
+
+ @Override
+ public Map<JobId, Job> getAllJobs() {
+ return jobs; // OK
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public EventHandler getEventHandler() {
+ return null;
+ }
+
+ @Override
+ public Clock getClock() {
+ return null;
+ }
+
+ @Override
+ public String getApplicationName() {
+ return "TestApp";
+ }
+
+ @Override
+ public long getStartTime() {
+ return startTime;
+ }
+
+ @Override
+ public ClusterInfo getClusterInfo() {
+ return null;
+ }
+ }
+
+ private Injector injector = Guice.createInjector(new ServletModule() {
+ @Override
+ protected void configureServlets() {
+
+ appContext = new TestAppContext();
+ bind(JAXBContextResolver.class);
+ bind(AMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ bind(AppContext.class).toInstance(appContext);
+ bind(Configuration.class).toInstance(conf);
+
+ serve("/*").with(GuiceContainer.class);
+ }
+ });
+
+ public class GuiceServletConfig extends GuiceServletContextListener {
+
+ @Override
+ protected Injector getInjector() {
+ return injector;
+ }
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ }
+
+ public TestAMWebServicesAttempts() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.mapreduce.v2.app2.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testTaskAttempts() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid).path("attempts")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ verifyAMTaskAttempts(json, task);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskAttemptsSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid).path("attempts/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ verifyAMTaskAttempts(json, task);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskAttemptsDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid).path("attempts")
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ verifyAMTaskAttempts(json, task);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskAttemptsXML() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid).path("attempts")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList attempts = dom.getElementsByTagName("taskAttempts");
+ assertEquals("incorrect number of elements", 1, attempts.getLength());
+
+ NodeList nodes = dom.getElementsByTagName("taskAttempt");
+ verifyAMTaskAttemptsXML(nodes, task);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskAttemptId() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+ String tid = MRApps.toString(task.getID());
+
+ for (TaskAttempt att : task.getAttempts().values()) {
+ TaskAttemptId attemptid = att.getID();
+ String attid = MRApps.toString(attemptid);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid)
+ .path("attempts").path(attid).accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("taskAttempt");
+ verifyAMTaskAttempt(info, att, task.getType());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testTaskAttemptIdSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+ String tid = MRApps.toString(task.getID());
+
+ for (TaskAttempt att : task.getAttempts().values()) {
+ TaskAttemptId attemptid = att.getID();
+ String attid = MRApps.toString(attemptid);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid)
+ .path("attempts").path(attid + "/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("taskAttempt");
+ verifyAMTaskAttempt(info, att, task.getType());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testTaskAttemptIdDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+ String tid = MRApps.toString(task.getID());
+
+ for (TaskAttempt att : task.getAttempts().values()) {
+ TaskAttemptId attemptid = att.getID();
+ String attid = MRApps.toString(attemptid);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid)
+ .path("attempts").path(attid).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("taskAttempt");
+ verifyAMTaskAttempt(info, att, task.getType());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testTaskAttemptIdXML() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ for (TaskAttempt att : task.getAttempts().values()) {
+ TaskAttemptId attemptid = att.getID();
+ String attid = MRApps.toString(attemptid);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid)
+ .path("attempts").path(attid).accept(MediaType.APPLICATION_XML)
+ .get(ClientResponse.class);
+
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList nodes = dom.getElementsByTagName("taskAttempt");
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+ verifyAMTaskAttemptXML(element, att, task.getType());
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testTaskAttemptIdBogus() throws JSONException, Exception {
+
+ testTaskAttemptIdErrorGeneric("bogusid",
+ "java.lang.Exception: TaskAttemptId string : bogusid is not properly formed");
+ }
+
+ @Test
+ public void testTaskAttemptIdNonExist() throws JSONException, Exception {
+
+ testTaskAttemptIdErrorGeneric(
+ "attempt_0_12345_m_000000_0",
+ "java.lang.Exception: Error getting info on task attempt id attempt_0_12345_m_000000_0");
+ }
+
+ @Test
+ public void testTaskAttemptIdInvalid() throws JSONException, Exception {
+
+ testTaskAttemptIdErrorGeneric("attempt_0_12345_d_000000_0",
+ "java.lang.Exception: Bad TaskType identifier. TaskAttemptId string : attempt_0_12345_d_000000_0 is not properly formed.");
+ }
+
+ @Test
+ public void testTaskAttemptIdInvalid2() throws JSONException, Exception {
+
+ testTaskAttemptIdErrorGeneric("attempt_12345_m_000000_0",
+ "java.lang.Exception: TaskAttemptId string : attempt_12345_m_000000_0 is not properly formed");
+ }
+
+ @Test
+ public void testTaskAttemptIdInvalid3() throws JSONException, Exception {
+
+ testTaskAttemptIdErrorGeneric("attempt_0_12345_m_000000",
+ "java.lang.Exception: TaskAttemptId string : attempt_0_12345_m_000000 is not properly formed");
+ }
+
+ private void testTaskAttemptIdErrorGeneric(String attid, String error)
+ throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+ String tid = MRApps.toString(task.getID());
+
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
+ .path("tasks").path(tid).path("attempts").path(attid)
+ .accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringMatch("exception message", error,
+ message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "NotFoundException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ }
+ }
+ }
+ }
+
+ public void verifyAMTaskAttemptXML(Element element, TaskAttempt att,
+ TaskType ttype) {
+ verifyTaskAttemptGeneric(att, ttype,
+ WebServicesTestUtils.getXmlString(element, "id"),
+ WebServicesTestUtils.getXmlString(element, "state"),
+ WebServicesTestUtils.getXmlString(element, "type"),
+ WebServicesTestUtils.getXmlString(element, "rack"),
+ WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
+ WebServicesTestUtils.getXmlString(element, "diagnostics"),
+ WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
+ WebServicesTestUtils.getXmlLong(element, "startTime"),
+ WebServicesTestUtils.getXmlLong(element, "finishTime"),
+ WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
+ WebServicesTestUtils.getXmlFloat(element, "progress"));
+
+ if (ttype == TaskType.REDUCE) {
+ verifyReduceTaskAttemptGeneric(att,
+ WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
+ WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
+ WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
+ WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
+ WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
+ }
+ }
+
+ public void verifyAMTaskAttempt(JSONObject info, TaskAttempt att,
+ TaskType ttype) throws JSONException {
+ if (ttype == TaskType.REDUCE) {
+ assertEquals("incorrect number of elements", 16, info.length());
+ } else {
+ assertEquals("incorrect number of elements", 11, info.length());
+ }
+
+ verifyTaskAttemptGeneric(att, ttype, info.getString("id"),
+ info.getString("state"), info.getString("type"),
+ info.getString("rack"), info.getString("nodeHttpAddress"),
+ info.getString("diagnostics"), info.getString("assignedContainerId"),
+ info.getLong("startTime"), info.getLong("finishTime"),
+ info.getLong("elapsedTime"), (float) info.getDouble("progress"));
+
+ if (ttype == TaskType.REDUCE) {
+ verifyReduceTaskAttemptGeneric(att, info.getLong("shuffleFinishTime"),
+ info.getLong("mergeFinishTime"), info.getLong("elapsedShuffleTime"),
+ info.getLong("elapsedMergeTime"), info.getLong("elapsedReduceTime"));
+ }
+ }
+
+ public void verifyAMTaskAttempts(JSONObject json, Task task)
+ throws JSONException {
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject attempts = json.getJSONObject("taskAttempts");
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONArray arr = attempts.getJSONArray("taskAttempt");
+ for (TaskAttempt att : task.getAttempts().values()) {
+ TaskAttemptId id = att.getID();
+ String attid = MRApps.toString(id);
+ Boolean found = false;
+
+ for (int i = 0; i < arr.length(); i++) {
+ JSONObject info = arr.getJSONObject(i);
+ if (attid.matches(info.getString("id"))) {
+ found = true;
+ verifyAMTaskAttempt(info, att, task.getType());
+ }
+ }
+ assertTrue("task attempt with id: " + attid
+ + " not in web service output", found);
+ }
+ }
+
+ public void verifyAMTaskAttemptsXML(NodeList nodes, Task task) {
+ assertEquals("incorrect number of elements", 1, nodes.getLength());
+
+ for (TaskAttempt att : task.getAttempts().values()) {
+ TaskAttemptId id = att.getID();
+ String attid = MRApps.toString(id);
+ Boolean found = false;
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+
+ if (attid.matches(WebServicesTestUtils.getXmlString(element, "id"))) {
+ found = true;
+ verifyAMTaskAttemptXML(element, att, task.getType());
+ }
+ }
+ assertTrue("task with id: " + attid + " not in web service output", found);
+ }
+ }
+
+ public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype,
+ String id, String state, String type, String rack,
+ String nodeHttpAddress, String diagnostics, String assignedContainerId,
+ long startTime, long finishTime, long elapsedTime, float progress) {
+
+ TaskAttemptId attid = ta.getID();
+ String attemptId = MRApps.toString(attid);
+
+ WebServicesTestUtils.checkStringMatch("id", attemptId, id);
+ WebServicesTestUtils.checkStringMatch("type", ttype.toString(), type);
+ WebServicesTestUtils.checkStringMatch("state", ta.getState().toString(),
+ state);
+ WebServicesTestUtils.checkStringMatch("rack", ta.getNodeRackName(), rack);
+ WebServicesTestUtils.checkStringMatch("nodeHttpAddress",
+ ta.getNodeHttpAddress(), nodeHttpAddress);
+
+ String expectDiag = "";
+ List<String> diagnosticsList = ta.getDiagnostics();
+ if (diagnosticsList != null && !diagnostics.isEmpty()) {
+ StringBuffer b = new StringBuffer();
+ for (String diag : diagnosticsList) {
+ b.append(diag);
+ }
+ expectDiag = b.toString();
+ }
+ WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
+ diagnostics);
+ WebServicesTestUtils.checkStringMatch("assignedContainerId",
+ ConverterUtils.toString(ta.getAssignedContainerID()),
+ assignedContainerId);
+
+ assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
+ assertEquals("finishTime wrong", ta.getFinishTime(), finishTime);
+ assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
+ assertEquals("progress wrong", ta.getProgress() * 100, progress, 1e-3f);
+ }
+
+ public void verifyReduceTaskAttemptGeneric(TaskAttempt ta,
+ long shuffleFinishTime, long mergeFinishTime, long elapsedShuffleTime,
+ long elapsedMergeTime, long elapsedReduceTime) {
+
+ assertEquals("shuffleFinishTime wrong", ta.getShuffleFinishTime(),
+ shuffleFinishTime);
+ assertEquals("mergeFinishTime wrong", ta.getSortFinishTime(),
+ mergeFinishTime);
+ assertEquals("elapsedShuffleTime wrong",
+ ta.getLaunchTime() - ta.getShuffleFinishTime(), elapsedShuffleTime);
+ assertEquals("elapsedMergeTime wrong",
+ ta.getShuffleFinishTime() - ta.getSortFinishTime(), elapsedMergeTime);
+ assertEquals("elapsedReduceTime wrong",
+ ta.getSortFinishTime() - ta.getFinishTime(), elapsedReduceTime);
+ }
+
+ @Test
+ public void testTaskAttemptIdCounters() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+ String tid = MRApps.toString(task.getID());
+
+ for (TaskAttempt att : task.getAttempts().values()) {
+ TaskAttemptId attemptid = att.getID();
+ String attid = MRApps.toString(attemptid);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid)
+ .path("attempts").path(attid).path("counters")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobTaskAttemptCounters");
+ verifyAMJobTaskAttemptCounters(info, att);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testTaskAttemptIdXMLCounters() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ for (TaskAttempt att : task.getAttempts().values()) {
+ TaskAttemptId attemptid = att.getID();
+ String attid = MRApps.toString(attemptid);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid)
+ .path("attempts").path(attid).path("counters")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList nodes = dom.getElementsByTagName("jobTaskAttemptCounters");
+
+ verifyAMTaskCountersXML(nodes, att);
+ }
+ }
+ }
+ }
+
+ public void verifyAMJobTaskAttemptCounters(JSONObject info, TaskAttempt att)
+ throws JSONException {
+
+ assertEquals("incorrect number of elements", 2, info.length());
+
+ WebServicesTestUtils.checkStringMatch("id", MRApps.toString(att.getID()),
+ info.getString("id"));
+
+ // just do simple verification of fields - not data is correct
+ // in the fields
+ JSONArray counterGroups = info.getJSONArray("taskAttemptCounterGroup");
+ for (int i = 0; i < counterGroups.length(); i++) {
+ JSONObject counterGroup = counterGroups.getJSONObject(i);
+ String name = counterGroup.getString("counterGroupName");
+ assertTrue("name not set", (name != null && !name.isEmpty()));
+ JSONArray counters = counterGroup.getJSONArray("counter");
+ for (int j = 0; j < counters.length(); j++) {
+ JSONObject counter = counters.getJSONObject(j);
+ String counterName = counter.getString("name");
+ assertTrue("name not set",
+ (counterName != null && !counterName.isEmpty()));
+ long value = counter.getLong("value");
+ assertTrue("value >= 0", value >= 0);
+ }
+ }
+ }
+
+ public void verifyAMTaskCountersXML(NodeList nodes, TaskAttempt att) {
+
+ for (int i = 0; i < nodes.getLength(); i++) {
+
+ Element element = (Element) nodes.item(i);
+ WebServicesTestUtils.checkStringMatch("id", MRApps.toString(att.getID()),
+ WebServicesTestUtils.getXmlString(element, "id"));
+ // just do simple verification of fields - not data is correct
+ // in the fields
+ NodeList groups = element.getElementsByTagName("taskAttemptCounterGroup");
+
+ for (int j = 0; j < groups.getLength(); j++) {
+ Element counters = (Element) groups.item(j);
+ assertNotNull("should have counters in the web service info", counters);
+ String name = WebServicesTestUtils.getXmlString(counters,
+ "counterGroupName");
+ assertTrue("name not set", (name != null && !name.isEmpty()));
+ NodeList counterArr = counters.getElementsByTagName("counter");
+ for (int z = 0; z < counterArr.getLength(); z++) {
+ Element counter = (Element) counterArr.item(z);
+ String counterName = WebServicesTestUtils.getXmlString(counter,
+ "name");
+ assertTrue("counter name not set",
+ (counterName != null && !counterName.isEmpty()));
+
+ long value = WebServicesTestUtils.getXmlLong(counter, "value");
+ assertTrue("value not >= 0", value >= 0);
+
+ }
+ }
+ }
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesJobConf.java
index 459aef1..108ff56 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesJobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesJobConf.java
@@ -1,342 +1,342 @@
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.hadoop.mapreduce.v2.app2.webapp;
-//
-//import static org.junit.Assert.assertEquals;
-//import static org.junit.Assert.assertNotNull;
-//import static org.junit.Assert.assertTrue;
-//import static org.junit.Assert.fail;
-//
-//import java.io.File;
-//import java.io.IOException;
-//import java.io.OutputStream;
-//import java.io.StringReader;
-//import java.util.Map;
-//
-//import javax.ws.rs.core.MediaType;
-//import javax.xml.parsers.DocumentBuilder;
-//import javax.xml.parsers.DocumentBuilderFactory;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.fs.FileSystem;
-//import org.apache.hadoop.fs.FileUtil;
-//import org.apache.hadoop.fs.Path;
-//import org.apache.hadoop.mapreduce.MRJobConfig;
-//import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-//import org.apache.hadoop.mapreduce.v2.app2.AppContext;
-//import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Job;
-//import org.apache.hadoop.mapreduce.v2.util.MRApps;
-//import org.apache.hadoop.yarn.Clock;
-//import org.apache.hadoop.yarn.ClusterInfo;
-//import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-//import org.apache.hadoop.yarn.api.records.ApplicationId;
-//import org.apache.hadoop.yarn.event.EventHandler;
-//import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-//import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
-//import org.codehaus.jettison.json.JSONArray;
-//import org.codehaus.jettison.json.JSONException;
-//import org.codehaus.jettison.json.JSONObject;
-//import org.junit.AfterClass;
-//import org.junit.Before;
-//import org.junit.Test;
-//import org.w3c.dom.Document;
-//import org.w3c.dom.Element;
-//import org.w3c.dom.NodeList;
-//import org.xml.sax.InputSource;
-//
-//import com.google.common.collect.Maps;
-//import com.google.inject.Guice;
-//import com.google.inject.Injector;
-//import com.google.inject.servlet.GuiceServletContextListener;
-//import com.google.inject.servlet.ServletModule;
-//import com.sun.jersey.api.client.ClientResponse;
-//import com.sun.jersey.api.client.WebResource;
-//import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-//import com.sun.jersey.test.framework.JerseyTest;
-//import com.sun.jersey.test.framework.WebAppDescriptor;
-//
-///**
-// * Test the app master web service Rest API for getting the job conf. This
-// * requires created a temporary configuration file.
-// *
-// * /ws/v1/mapreduce/job/{jobid}/conf
-// */
-//public class TestAMWebServicesJobConf extends JerseyTest {
-//
-// private static Configuration conf = new Configuration();
-// private static TestAppContext appContext;
-//
-// private static File testConfDir = new File("target",
-// TestAMWebServicesJobConf.class.getSimpleName() + "confDir");
-//
-// static class TestAppContext implements AppContext {
-// final ApplicationAttemptId appAttemptID;
-// final ApplicationId appID;
-// final String user = MockJobs.newUserName();
-// final Map<JobId, Job> jobs;
-// final long startTime = System.currentTimeMillis();
-//
-// TestAppContext(int appid, int numTasks, int numAttempts, Path confPath) {
-// appID = MockJobs.newAppID(appid);
-// appAttemptID = MockJobs.newAppAttemptID(appID, 0);
-// Map<JobId, Job> map = Maps.newHashMap();
-// Job job = MockJobs.newJob(appID, 0, numTasks, numAttempts, confPath);
-// map.put(job.getID(), job);
-// jobs = map;
-// }
-//
-// @Override
-// public ApplicationAttemptId getApplicationAttemptId() {
-// return appAttemptID;
-// }
-//
-// @Override
-// public ApplicationId getApplicationID() {
-// return appID;
-// }
-//
-// @Override
-// public CharSequence getUser() {
-// return user;
-// }
-//
-// @Override
-// public Job getJob(JobId jobID) {
-// return jobs.get(jobID);
-// }
-//
-// @Override
-// public Map<JobId, Job> getAllJobs() {
-// return jobs; // OK
-// }
-//
-// @SuppressWarnings("rawtypes")
-// @Override
-// public EventHandler getEventHandler() {
-// return null;
-// }
-//
-// @Override
-// public Clock getClock() {
-// return null;
-// }
-//
-// @Override
-// public String getApplicationName() {
-// return "TestApp";
-// }
-//
-// @Override
-// public long getStartTime() {
-// return startTime;
-// }
-//
-// @Override
-// public ClusterInfo getClusterInfo() {
-// return null;
-// }
-// }
-//
-// private Injector injector = Guice.createInjector(new ServletModule() {
-// @Override
-// protected void configureServlets() {
-//
-// Path confPath = new Path(testConfDir.toString(),
-// MRJobConfig.JOB_CONF_FILE);
-// Configuration config = new Configuration();
-//
-// FileSystem localFs;
-// try {
-// localFs = FileSystem.getLocal(config);
-// confPath = localFs.makeQualified(confPath);
-//
-// OutputStream out = localFs.create(confPath);
-// try {
-// conf.writeXml(out);
-// } finally {
-// out.close();
-// }
-// if (!localFs.exists(confPath)) {
-// fail("error creating config file: " + confPath);
-// }
-//
-// } catch (IOException e) {
-// fail("error creating config file: " + e.getMessage());
-// }
-//
-// appContext = new TestAppContext(0, 2, 1, confPath);
-//
-// bind(JAXBContextResolver.class);
-// bind(AMWebServices.class);
-// bind(GenericExceptionHandler.class);
-// bind(AppContext.class).toInstance(appContext);
-// bind(Configuration.class).toInstance(conf);
-//
-// serve("/*").with(GuiceContainer.class);
-// }
-// });
-//
-// public class GuiceServletConfig extends GuiceServletContextListener {
-//
-// @Override
-// protected Injector getInjector() {
-// return injector;
-// }
-// }
-//
-// @Before
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-// testConfDir.mkdir();
-//
-// }
-//
-// @AfterClass
-// static public void stop() {
-// FileUtil.fullyDelete(testConfDir);
-// }
-//
-// public TestAMWebServicesJobConf() {
-// super(new WebAppDescriptor.Builder(
-// "org.apache.hadoop.mapreduce.v2.app2.webapp")
-// .contextListenerClass(GuiceServletConfig.class)
-// .filterClass(com.google.inject.servlet.GuiceFilter.class)
-// .contextPath("jersey-guice-filter").servletPath("/").build());
-// }
-//
-// @Test
-// public void testJobConf() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("conf")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("conf");
-// verifyAMJobConf(info, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testJobConfSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("conf/")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("conf");
-// verifyAMJobConf(info, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testJobConfDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("conf").get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("conf");
-// verifyAMJobConf(info, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testJobConfXML() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("conf")
-// .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList info = dom.getElementsByTagName("conf");
-// verifyAMJobConfXML(info, jobsMap.get(id));
-// }
-// }
-//
-// public void verifyAMJobConf(JSONObject info, Job job) throws JSONException {
-//
-// assertEquals("incorrect number of elements", 2, info.length());
-//
-// WebServicesTestUtils.checkStringMatch("path", job.getConfFile().toString(),
-// info.getString("path"));
-// // just do simple verification of fields - not data is correct
-// // in the fields
-// JSONArray properties = info.getJSONArray("property");
-// for (int i = 0; i < properties.length(); i++) {
-// JSONObject prop = properties.getJSONObject(i);
-// String name = prop.getString("name");
-// String value = prop.getString("value");
-// assertTrue("name not set", (name != null && !name.isEmpty()));
-// assertTrue("value not set", (value != null && !value.isEmpty()));
-// }
-// }
-//
-// public void verifyAMJobConfXML(NodeList nodes, Job job) {
-//
-// assertEquals("incorrect number of elements", 1, nodes.getLength());
-//
-// for (int i = 0; i < nodes.getLength(); i++) {
-// Element element = (Element) nodes.item(i);
-// WebServicesTestUtils.checkStringMatch("path", job.getConfFile()
-// .toString(), WebServicesTestUtils.getXmlString(element, "path"));
-//
-// // just do simple verification of fields - not data is correct
-// // in the fields
-// NodeList properties = element.getElementsByTagName("property");
-//
-// for (int j = 0; j < properties.getLength(); j++) {
-// Element property = (Element) properties.item(j);
-// assertNotNull("should have counters in the web service info", property);
-// String name = WebServicesTestUtils.getXmlString(property, "name");
-// String value = WebServicesTestUtils.getXmlString(property, "value");
-// assertTrue("name not set", (name != null && !name.isEmpty()));
-// assertTrue("name not set", (value != null && !value.isEmpty()));
-// }
-// }
-// }
-//
-//}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.app2.webapp;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringReader;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.app2.AppContext;
+import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
+import org.apache.hadoop.mapreduce.v2.app2.job.Job;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.ClusterInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import com.google.common.collect.Maps;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+/**
+ * Test the app master web service Rest API for getting the job conf. This
+ * requires created a temporary configuration file.
+ *
+ * /ws/v1/mapreduce/job/{jobid}/conf
+ */
+public class TestAMWebServicesJobConf extends JerseyTest {
+
+ private static Configuration conf = new Configuration();
+ private static TestAppContext appContext;
+
+ private static File testConfDir = new File("target",
+ TestAMWebServicesJobConf.class.getSimpleName() + "confDir");
+
+ static class TestAppContext implements AppContext {
+ final ApplicationAttemptId appAttemptID;
+ final ApplicationId appID;
+ final String user = MockJobs.newUserName();
+ final Map<JobId, Job> jobs;
+ final long startTime = System.currentTimeMillis();
+
+ TestAppContext(int appid, int numTasks, int numAttempts, Path confPath) {
+ appID = MockJobs.newAppID(appid);
+ appAttemptID = MockJobs.newAppAttemptID(appID, 0);
+ Map<JobId, Job> map = Maps.newHashMap();
+ Job job = MockJobs.newJob(appID, 0, numTasks, numAttempts, confPath);
+ map.put(job.getID(), job);
+ jobs = map;
+ }
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptID;
+ }
+
+ @Override
+ public ApplicationId getApplicationID() {
+ return appID;
+ }
+
+ @Override
+ public CharSequence getUser() {
+ return user;
+ }
+
+ @Override
+ public Job getJob(JobId jobID) {
+ return jobs.get(jobID);
+ }
+
+ @Override
+ public Map<JobId, Job> getAllJobs() {
+ return jobs; // OK
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public EventHandler getEventHandler() {
+ return null;
+ }
+
+ @Override
+ public Clock getClock() {
+ return null;
+ }
+
+ @Override
+ public String getApplicationName() {
+ return "TestApp";
+ }
+
+ @Override
+ public long getStartTime() {
+ return startTime;
+ }
+
+ @Override
+ public ClusterInfo getClusterInfo() {
+ return null;
+ }
+ }
+
+ private Injector injector = Guice.createInjector(new ServletModule() {
+ @Override
+ protected void configureServlets() {
+
+ Path confPath = new Path(testConfDir.toString(),
+ MRJobConfig.JOB_CONF_FILE);
+ Configuration config = new Configuration();
+
+ FileSystem localFs;
+ try {
+ localFs = FileSystem.getLocal(config);
+ confPath = localFs.makeQualified(confPath);
+
+ OutputStream out = localFs.create(confPath);
+ try {
+ conf.writeXml(out);
+ } finally {
+ out.close();
+ }
+ if (!localFs.exists(confPath)) {
+ fail("error creating config file: " + confPath);
+ }
+
+ } catch (IOException e) {
+ fail("error creating config file: " + e.getMessage());
+ }
+
+ appContext = new TestAppContext(0, 2, 1, confPath);
+
+ bind(JAXBContextResolver.class);
+ bind(AMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ bind(AppContext.class).toInstance(appContext);
+ bind(Configuration.class).toInstance(conf);
+
+ serve("/*").with(GuiceContainer.class);
+ }
+ });
+
+ public class GuiceServletConfig extends GuiceServletContextListener {
+
+ @Override
+ protected Injector getInjector() {
+ return injector;
+ }
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ testConfDir.mkdir();
+
+ }
+
+ @AfterClass
+ static public void stop() {
+ FileUtil.fullyDelete(testConfDir);
+ }
+
+ public TestAMWebServicesJobConf() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.mapreduce.v2.app2.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testJobConf() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("conf")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("conf");
+ verifyAMJobConf(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobConfSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("conf/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("conf");
+ verifyAMJobConf(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobConfDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("conf").get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("conf");
+ verifyAMJobConf(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobConfXML() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("conf")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList info = dom.getElementsByTagName("conf");
+ verifyAMJobConfXML(info, jobsMap.get(id));
+ }
+ }
+
+ public void verifyAMJobConf(JSONObject info, Job job) throws JSONException {
+
+ assertEquals("incorrect number of elements", 2, info.length());
+
+ WebServicesTestUtils.checkStringMatch("path", job.getConfFile().toString(),
+ info.getString("path"));
+ // just do simple verification of fields - not data is correct
+ // in the fields
+ JSONArray properties = info.getJSONArray("property");
+ for (int i = 0; i < properties.length(); i++) {
+ JSONObject prop = properties.getJSONObject(i);
+ String name = prop.getString("name");
+ String value = prop.getString("value");
+ assertTrue("name not set", (name != null && !name.isEmpty()));
+ assertTrue("value not set", (value != null && !value.isEmpty()));
+ }
+ }
+
+ public void verifyAMJobConfXML(NodeList nodes, Job job) {
+
+ assertEquals("incorrect number of elements", 1, nodes.getLength());
+
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+ WebServicesTestUtils.checkStringMatch("path", job.getConfFile()
+ .toString(), WebServicesTestUtils.getXmlString(element, "path"));
+
+ // just do simple verification of fields - not data is correct
+ // in the fields
+ NodeList properties = element.getElementsByTagName("property");
+
+ for (int j = 0; j < properties.getLength(); j++) {
+ Element property = (Element) properties.item(j);
+ assertNotNull("should have counters in the web service info", property);
+ String name = WebServicesTestUtils.getXmlString(property, "name");
+ String value = WebServicesTestUtils.getXmlString(property, "value");
+ assertTrue("name not set", (name != null && !name.isEmpty()));
+ assertTrue("name not set", (value != null && !value.isEmpty()));
+ }
+ }
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesJobs.java
index 520abf5..31ed4db 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesJobs.java
@@ -1,984 +1,984 @@
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.hadoop.mapreduce.v2.app2.webapp;
-//
-//import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
-//import static org.junit.Assert.assertEquals;
-//import static org.junit.Assert.assertNotNull;
-//import static org.junit.Assert.assertTrue;
-//import static org.junit.Assert.fail;
-//
-//import java.io.StringReader;
-//import java.util.List;
-//import java.util.Map;
-//
-//import javax.ws.rs.core.MediaType;
-//import javax.xml.parsers.DocumentBuilder;
-//import javax.xml.parsers.DocumentBuilderFactory;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.mapreduce.JobACL;
-//import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
-//import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-//import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
-//import org.apache.hadoop.mapreduce.v2.app2.AppContext;
-//import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Job;
-//import org.apache.hadoop.mapreduce.v2.util.MRApps;
-//import org.apache.hadoop.security.authorize.AccessControlList;
-//import org.apache.hadoop.yarn.Clock;
-//import org.apache.hadoop.yarn.ClusterInfo;
-//import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-//import org.apache.hadoop.yarn.api.records.ApplicationId;
-//import org.apache.hadoop.yarn.event.EventHandler;
-//import org.apache.hadoop.yarn.util.BuilderUtils;
-//import org.apache.hadoop.yarn.util.Times;
-//import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-//import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
-//import org.codehaus.jettison.json.JSONArray;
-//import org.codehaus.jettison.json.JSONException;
-//import org.codehaus.jettison.json.JSONObject;
-//import org.junit.Before;
-//import org.junit.Test;
-//import org.w3c.dom.Document;
-//import org.w3c.dom.Element;
-//import org.w3c.dom.NodeList;
-//import org.xml.sax.InputSource;
-//
-//import com.google.inject.Guice;
-//import com.google.inject.Injector;
-//import com.google.inject.servlet.GuiceServletContextListener;
-//import com.google.inject.servlet.ServletModule;
-//import com.sun.jersey.api.client.ClientResponse;
-//import com.sun.jersey.api.client.UniformInterfaceException;
-//import com.sun.jersey.api.client.WebResource;
-//import com.sun.jersey.api.client.ClientResponse.Status;
-//import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-//import com.sun.jersey.test.framework.JerseyTest;
-//import com.sun.jersey.test.framework.WebAppDescriptor;
-//
-///**
-// * Test the app master web service Rest API for getting jobs, a specific job,
-// * and job counters.
-// *
-// * /ws/v1/mapreduce/jobs
-// * /ws/v1/mapreduce/jobs/{jobid}
-// * /ws/v1/mapreduce/jobs/{jobid}/counters
-// * /ws/v1/mapreduce/jobs/{jobid}/jobattempts
-// */
-//public class TestAMWebServicesJobs extends JerseyTest {
-//
-// private static Configuration conf = new Configuration();
-// private static TestAppContext appContext;
-//
-// static class TestAppContext implements AppContext {
-// final ApplicationAttemptId appAttemptID;
-// final ApplicationId appID;
-// final String user = MockJobs.newUserName();
-// final Map<JobId, Job> jobs;
-// final long startTime = System.currentTimeMillis();
-//
-// TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
-// appID = MockJobs.newAppID(appid);
-// appAttemptID = MockJobs.newAppAttemptID(appID, 0);
-// jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
-// }
-//
-// TestAppContext() {
-// this(0, 1, 2, 1);
-// }
-//
-// @Override
-// public ApplicationAttemptId getApplicationAttemptId() {
-// return appAttemptID;
-// }
-//
-// @Override
-// public ApplicationId getApplicationID() {
-// return appID;
-// }
-//
-// @Override
-// public CharSequence getUser() {
-// return user;
-// }
-//
-// @Override
-// public Job getJob(JobId jobID) {
-// return jobs.get(jobID);
-// }
-//
-// @Override
-// public Map<JobId, Job> getAllJobs() {
-// return jobs; // OK
-// }
-//
-// @SuppressWarnings("rawtypes")
-// @Override
-// public EventHandler getEventHandler() {
-// return null;
-// }
-//
-// @Override
-// public Clock getClock() {
-// return null;
-// }
-//
-// @Override
-// public String getApplicationName() {
-// return "TestApp";
-// }
-//
-// @Override
-// public long getStartTime() {
-// return startTime;
-// }
-//
-// @Override
-// public ClusterInfo getClusterInfo() {
-// return null;
-// }
-// }
-//
-// private Injector injector = Guice.createInjector(new ServletModule() {
-// @Override
-// protected void configureServlets() {
-//
-// appContext = new TestAppContext();
-// bind(JAXBContextResolver.class);
-// bind(AMWebServices.class);
-// bind(GenericExceptionHandler.class);
-// bind(AppContext.class).toInstance(appContext);
-// bind(Configuration.class).toInstance(conf);
-//
-// serve("/*").with(GuiceContainer.class);
-// }
-// });
-//
-// public class GuiceServletConfig extends GuiceServletContextListener {
-//
-// @Override
-// protected Injector getInjector() {
-// return injector;
-// }
-// }
-//
-// @Before
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// }
-//
-// public TestAMWebServicesJobs() {
-// super(new WebAppDescriptor.Builder(
-// "org.apache.hadoop.mapreduce.v2.app2.webapp")
-// .contextListenerClass(GuiceServletConfig.class)
-// .filterClass(com.google.inject.servlet.GuiceFilter.class)
-// .contextPath("jersey-guice-filter").servletPath("/").build());
-// }
-//
-// @Test
-// public void testJobs() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").accept(MediaType.APPLICATION_JSON)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject jobs = json.getJSONObject("jobs");
-// JSONArray arr = jobs.getJSONArray("job");
-// JSONObject info = arr.getJSONObject(0);
-// Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
-// verifyAMJob(info, job);
-//
-// }
-//
-// @Test
-// public void testJobsSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs/").accept(MediaType.APPLICATION_JSON)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject jobs = json.getJSONObject("jobs");
-// JSONArray arr = jobs.getJSONArray("job");
-// JSONObject info = arr.getJSONObject(0);
-// Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
-// verifyAMJob(info, job);
-//
-// }
-//
-// @Test
-// public void testJobsDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject jobs = json.getJSONObject("jobs");
-// JSONArray arr = jobs.getJSONArray("job");
-// JSONObject info = arr.getJSONObject(0);
-// Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
-// verifyAMJob(info, job);
-//
-// }
-//
-// @Test
-// public void testJobsXML() throws Exception {
-// WebResource r = resource();
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").accept(MediaType.APPLICATION_XML)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList jobs = dom.getElementsByTagName("jobs");
-// assertEquals("incorrect number of elements", 1, jobs.getLength());
-// NodeList job = dom.getElementsByTagName("job");
-// assertEquals("incorrect number of elements", 1, job.getLength());
-// verifyAMJobXML(job, appContext);
-//
-// }
-//
-// @Test
-// public void testJobId() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).accept(MediaType.APPLICATION_JSON)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("job");
-// verifyAMJob(info, jobsMap.get(id));
-// }
-//
-// }
-//
-// @Test
-// public void testJobIdSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId + "/").accept(MediaType.APPLICATION_JSON)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("job");
-// verifyAMJob(info, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testJobIdDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("job");
-// verifyAMJob(info, jobsMap.get(id));
-// }
-//
-// }
-//
-// @Test
-// public void testJobIdNonExist() throws JSONException, Exception {
-// WebResource r = resource();
-//
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs")
-// .path("job_0_1234").get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// WebServicesTestUtils.checkStringMatch("exception message",
-// "java.lang.Exception: job, job_0_1234, is not found", message);
-// WebServicesTestUtils.checkStringMatch("exception type",
-// "NotFoundException", type);
-// WebServicesTestUtils.checkStringMatch("exception classname",
-// "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
-// }
-// }
-//
-// @Test
-// public void testJobIdInvalid() throws JSONException, Exception {
-// WebResource r = resource();
-//
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path("job_foo")
-// .accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// verifyJobIdInvalid(message, type, classname);
-// }
-// }
-//
-// // verify the exception output default is JSON
-// @Test
-// public void testJobIdInvalidDefault() throws JSONException, Exception {
-// WebResource r = resource();
-//
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path("job_foo")
-// .get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// verifyJobIdInvalid(message, type, classname);
-// }
-// }
-//
-// // test that the exception output works in XML
-// @Test
-// public void testJobIdInvalidXML() throws JSONException, Exception {
-// WebResource r = resource();
-//
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path("job_foo")
-// .accept(MediaType.APPLICATION_XML).get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String msg = response.getEntity(String.class);
-// System.out.println(msg);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(msg));
-// Document dom = db.parse(is);
-// NodeList nodes = dom.getElementsByTagName("RemoteException");
-// Element element = (Element) nodes.item(0);
-// String message = WebServicesTestUtils.getXmlString(element, "message");
-// String type = WebServicesTestUtils.getXmlString(element, "exception");
-// String classname = WebServicesTestUtils.getXmlString(element,
-// "javaClassName");
-// verifyJobIdInvalid(message, type, classname);
-// }
-// }
-//
-// private void verifyJobIdInvalid(String message, String type, String classname) {
-// WebServicesTestUtils.checkStringMatch("exception message",
-// "java.lang.Exception: JobId string : job_foo is not properly formed",
-// message);
-// WebServicesTestUtils.checkStringMatch("exception type",
-// "NotFoundException", type);
-// WebServicesTestUtils.checkStringMatch("exception classname",
-// "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
-// }
-//
-// @Test
-// public void testJobIdInvalidBogus() throws JSONException, Exception {
-// WebResource r = resource();
-//
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path("bogusfoo")
-// .get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// WebServicesTestUtils
-// .checkStringMatch(
-// "exception message",
-// "java.lang.Exception: JobId string : bogusfoo is not properly formed",
-// message);
-// WebServicesTestUtils.checkStringMatch("exception type",
-// "NotFoundException", type);
-// WebServicesTestUtils.checkStringMatch("exception classname",
-// "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
-// }
-// }
-//
-// @Test
-// public void testJobIdXML() throws Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).accept(MediaType.APPLICATION_XML)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList job = dom.getElementsByTagName("job");
-// verifyAMJobXML(job, appContext);
-// }
-//
-// }
-//
-// public void verifyAMJob(JSONObject info, Job job) throws JSONException {
-//
-// assertEquals("incorrect number of elements", 30, info.length());
-//
-// // everyone access fields
-// verifyAMJobGeneric(job, info.getString("id"), info.getString("user"),
-// info.getString("name"), info.getString("state"),
-// info.getLong("startTime"), info.getLong("finishTime"),
-// info.getLong("elapsedTime"), info.getInt("mapsTotal"),
-// info.getInt("mapsCompleted"), info.getInt("reducesTotal"),
-// info.getInt("reducesCompleted"),
-// (float) info.getDouble("reduceProgress"),
-// (float) info.getDouble("mapProgress"));
-//
-// String diagnostics = "";
-// if (info.has("diagnostics")) {
-// diagnostics = info.getString("diagnostics");
-// }
-//
-// // restricted access fields - if security and acls set
-// verifyAMJobGenericSecure(job, info.getInt("mapsPending"),
-// info.getInt("mapsRunning"), info.getInt("reducesPending"),
-// info.getInt("reducesRunning"), info.getBoolean("uberized"),
-// diagnostics, info.getInt("newReduceAttempts"),
-// info.getInt("runningReduceAttempts"),
-// info.getInt("failedReduceAttempts"),
-// info.getInt("killedReduceAttempts"),
-// info.getInt("successfulReduceAttempts"), info.getInt("newMapAttempts"),
-// info.getInt("runningMapAttempts"), info.getInt("failedMapAttempts"),
-// info.getInt("killedMapAttempts"), info.getInt("successfulMapAttempts"));
-//
-// Map<JobACL, AccessControlList> allacls = job.getJobACLs();
-// if (allacls != null) {
-//
-// for (Map.Entry<JobACL, AccessControlList> entry : allacls.entrySet()) {
-// String expectName = entry.getKey().getAclName();
-// String expectValue = entry.getValue().getAclString();
-// Boolean found = false;
-// // make sure ws includes it
-// if (info.has("acls")) {
-// JSONArray arr = info.getJSONArray("acls");
-//
-// for (int i = 0; i < arr.length(); i++) {
-// JSONObject aclInfo = arr.getJSONObject(i);
-// if (expectName.matches(aclInfo.getString("name"))) {
-// found = true;
-// WebServicesTestUtils.checkStringMatch("value", expectValue,
-// aclInfo.getString("value"));
-// }
-// }
-// } else {
-// fail("should have acls in the web service info");
-// }
-// assertTrue("acl: " + expectName + " not found in webservice output",
-// found);
-// }
-// }
-//
-// }
-//
-// public void verifyAMJobXML(NodeList nodes, TestAppContext appContext) {
-//
-// assertEquals("incorrect number of elements", 1, nodes.getLength());
-//
-// for (int i = 0; i < nodes.getLength(); i++) {
-// Element element = (Element) nodes.item(i);
-//
-// Job job = appContext.getJob(MRApps.toJobID(WebServicesTestUtils
-// .getXmlString(element, "id")));
-// assertNotNull("Job not found - output incorrect", job);
-//
-// verifyAMJobGeneric(job, WebServicesTestUtils.getXmlString(element, "id"),
-// WebServicesTestUtils.getXmlString(element, "user"),
-// WebServicesTestUtils.getXmlString(element, "name"),
-// WebServicesTestUtils.getXmlString(element, "state"),
-// WebServicesTestUtils.getXmlLong(element, "startTime"),
-// WebServicesTestUtils.getXmlLong(element, "finishTime"),
-// WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
-// WebServicesTestUtils.getXmlInt(element, "mapsTotal"),
-// WebServicesTestUtils.getXmlInt(element, "mapsCompleted"),
-// WebServicesTestUtils.getXmlInt(element, "reducesTotal"),
-// WebServicesTestUtils.getXmlInt(element, "reducesCompleted"),
-// WebServicesTestUtils.getXmlFloat(element, "reduceProgress"),
-// WebServicesTestUtils.getXmlFloat(element, "mapProgress"));
-//
-// // restricted access fields - if security and acls set
-// verifyAMJobGenericSecure(job,
-// WebServicesTestUtils.getXmlInt(element, "mapsPending"),
-// WebServicesTestUtils.getXmlInt(element, "mapsRunning"),
-// WebServicesTestUtils.getXmlInt(element, "reducesPending"),
-// WebServicesTestUtils.getXmlInt(element, "reducesRunning"),
-// WebServicesTestUtils.getXmlBoolean(element, "uberized"),
-// WebServicesTestUtils.getXmlString(element, "diagnostics"),
-// WebServicesTestUtils.getXmlInt(element, "newReduceAttempts"),
-// WebServicesTestUtils.getXmlInt(element, "runningReduceAttempts"),
-// WebServicesTestUtils.getXmlInt(element, "failedReduceAttempts"),
-// WebServicesTestUtils.getXmlInt(element, "killedReduceAttempts"),
-// WebServicesTestUtils.getXmlInt(element, "successfulReduceAttempts"),
-// WebServicesTestUtils.getXmlInt(element, "newMapAttempts"),
-// WebServicesTestUtils.getXmlInt(element, "runningMapAttempts"),
-// WebServicesTestUtils.getXmlInt(element, "failedMapAttempts"),
-// WebServicesTestUtils.getXmlInt(element, "killedMapAttempts"),
-// WebServicesTestUtils.getXmlInt(element, "successfulMapAttempts"));
-//
-// Map<JobACL, AccessControlList> allacls = job.getJobACLs();
-// if (allacls != null) {
-// for (Map.Entry<JobACL, AccessControlList> entry : allacls.entrySet()) {
-// String expectName = entry.getKey().getAclName();
-// String expectValue = entry.getValue().getAclString();
-// Boolean found = false;
-// // make sure ws includes it
-// NodeList id = element.getElementsByTagName("acls");
-// if (id != null) {
-// for (int j = 0; j < id.getLength(); j++) {
-// Element aclElem = (Element) id.item(j);
-// if (aclElem == null) {
-// fail("should have acls in the web service info");
-// }
-// if (expectName.matches(WebServicesTestUtils.getXmlString(aclElem,
-// "name"))) {
-// found = true;
-// WebServicesTestUtils.checkStringMatch("value", expectValue,
-// WebServicesTestUtils.getXmlString(aclElem, "value"));
-// }
-// }
-// } else {
-// fail("should have acls in the web service info");
-// }
-// assertTrue("acl: " + expectName + " not found in webservice output",
-// found);
-// }
-// }
-// }
-// }
-//
-// public void verifyAMJobGeneric(Job job, String id, String user, String name,
-// String state, long startTime, long finishTime, long elapsedTime,
-// int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
-// float reduceProgress, float mapProgress) {
-// JobReport report = job.getReport();
-//
-// WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
-// id);
-// WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
-// user);
-// WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
-// WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
-// state);
-//
-// assertEquals("startTime incorrect", report.getStartTime(), startTime);
-// assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
-// assertEquals("elapsedTime incorrect",
-// Times.elapsed(report.getStartTime(), report.getFinishTime()),
-// elapsedTime);
-// assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
-// assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
-// mapsCompleted);
-// assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
-// assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
-// reducesCompleted);
-// assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
-// mapProgress, 0);
-// assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
-// reduceProgress, 0);
-// }
-//
-// public void verifyAMJobGenericSecure(Job job, int mapsPending,
-// int mapsRunning, int reducesPending, int reducesRunning,
-// Boolean uberized, String diagnostics, int newReduceAttempts,
-// int runningReduceAttempts, int failedReduceAttempts,
-// int killedReduceAttempts, int successfulReduceAttempts,
-// int newMapAttempts, int runningMapAttempts, int failedMapAttempts,
-// int killedMapAttempts, int successfulMapAttempts) {
-//
-// String diagString = "";
-// List<String> diagList = job.getDiagnostics();
-// if (diagList != null && !diagList.isEmpty()) {
-// StringBuffer b = new StringBuffer();
-// for (String diag : diagList) {
-// b.append(diag);
-// }
-// diagString = b.toString();
-// }
-// WebServicesTestUtils.checkStringMatch("diagnostics", diagString,
-// diagnostics);
-//
-// assertEquals("isUber incorrect", job.isUber(), uberized);
-//
-// // unfortunately the following fields are all calculated in JobInfo
-// // so not easily accessible without doing all the calculations again.
-// // For now just make sure they are present.
-// assertTrue("mapsPending not >= 0", mapsPending >= 0);
-// assertTrue("mapsRunning not >= 0", mapsRunning >= 0);
-// assertTrue("reducesPending not >= 0", reducesPending >= 0);
-// assertTrue("reducesRunning not >= 0", reducesRunning >= 0);
-//
-// assertTrue("newReduceAttempts not >= 0", newReduceAttempts >= 0);
-// assertTrue("runningReduceAttempts not >= 0", runningReduceAttempts >= 0);
-// assertTrue("failedReduceAttempts not >= 0", failedReduceAttempts >= 0);
-// assertTrue("killedReduceAttempts not >= 0", killedReduceAttempts >= 0);
-// assertTrue("successfulReduceAttempts not >= 0",
-// successfulReduceAttempts >= 0);
-//
-// assertTrue("newMapAttempts not >= 0", newMapAttempts >= 0);
-// assertTrue("runningMapAttempts not >= 0", runningMapAttempts >= 0);
-// assertTrue("failedMapAttempts not >= 0", failedMapAttempts >= 0);
-// assertTrue("killedMapAttempts not >= 0", killedMapAttempts >= 0);
-// assertTrue("successfulMapAttempts not >= 0", successfulMapAttempts >= 0);
-//
-// }
-//
-// @Test
-// public void testJobCounters() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("counters")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("jobCounters");
-// verifyAMJobCounters(info, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testJobCountersSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("counters/")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("jobCounters");
-// verifyAMJobCounters(info, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testJobCountersDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("counters/").get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("jobCounters");
-// verifyAMJobCounters(info, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testJobCountersXML() throws Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("counters")
-// .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList info = dom.getElementsByTagName("jobCounters");
-// verifyAMJobCountersXML(info, jobsMap.get(id));
-// }
-// }
-//
-// public void verifyAMJobCounters(JSONObject info, Job job)
-// throws JSONException {
-//
-// assertEquals("incorrect number of elements", 2, info.length());
-//
-// WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
-// info.getString("id"));
-// // just do simple verification of fields - not data is correct
-// // in the fields
-// JSONArray counterGroups = info.getJSONArray("counterGroup");
-// for (int i = 0; i < counterGroups.length(); i++) {
-// JSONObject counterGroup = counterGroups.getJSONObject(i);
-// String name = counterGroup.getString("counterGroupName");
-// assertTrue("name not set", (name != null && !name.isEmpty()));
-// JSONArray counters = counterGroup.getJSONArray("counter");
-// for (int j = 0; j < counters.length(); j++) {
-// JSONObject counter = counters.getJSONObject(j);
-// String counterName = counter.getString("name");
-// assertTrue("counter name not set",
-// (counterName != null && !counterName.isEmpty()));
-//
-// long mapValue = counter.getLong("mapCounterValue");
-// assertTrue("mapCounterValue >= 0", mapValue >= 0);
-//
-// long reduceValue = counter.getLong("reduceCounterValue");
-// assertTrue("reduceCounterValue >= 0", reduceValue >= 0);
-//
-// long totalValue = counter.getLong("totalCounterValue");
-// assertTrue("totalCounterValue >= 0", totalValue >= 0);
-//
-// }
-// }
-// }
-//
-// public void verifyAMJobCountersXML(NodeList nodes, Job job) {
-//
-// for (int i = 0; i < nodes.getLength(); i++) {
-// Element element = (Element) nodes.item(i);
-//
-// assertNotNull("Job not found - output incorrect", job);
-//
-// WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
-// WebServicesTestUtils.getXmlString(element, "id"));
-// // just do simple verification of fields - not data is correct
-// // in the fields
-// NodeList groups = element.getElementsByTagName("counterGroup");
-//
-// for (int j = 0; j < groups.getLength(); j++) {
-// Element counters = (Element) groups.item(j);
-// assertNotNull("should have counters in the web service info", counters);
-// String name = WebServicesTestUtils.getXmlString(counters,
-// "counterGroupName");
-// assertTrue("name not set", (name != null && !name.isEmpty()));
-// NodeList counterArr = counters.getElementsByTagName("counter");
-// for (int z = 0; z < counterArr.getLength(); z++) {
-// Element counter = (Element) counterArr.item(z);
-// String counterName = WebServicesTestUtils.getXmlString(counter,
-// "name");
-// assertTrue("counter name not set",
-// (counterName != null && !counterName.isEmpty()));
-//
-// long mapValue = WebServicesTestUtils.getXmlLong(counter,
-// "mapCounterValue");
-// assertTrue("mapCounterValue not >= 0", mapValue >= 0);
-//
-// long reduceValue = WebServicesTestUtils.getXmlLong(counter,
-// "reduceCounterValue");
-// assertTrue("reduceCounterValue >= 0", reduceValue >= 0);
-//
-// long totalValue = WebServicesTestUtils.getXmlLong(counter,
-// "totalCounterValue");
-// assertTrue("totalCounterValue >= 0", totalValue >= 0);
-// }
-// }
-// }
-// }
-//
-// @Test
-// public void testJobAttempts() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1")
-// .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("jobAttempts");
-// verifyJobAttempts(info, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testJobAttemptsSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1")
-// .path("mapreduce").path("jobs").path(jobId).path("jobattempts/")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("jobAttempts");
-// verifyJobAttempts(info, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testJobAttemptsDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1")
-// .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("jobAttempts");
-// verifyJobAttempts(info, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testJobAttemptsXML() throws Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-//
-// ClientResponse response = r.path("ws").path("v1")
-// .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
-// .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList attempts = dom.getElementsByTagName("jobAttempts");
-// assertEquals("incorrect number of elements", 1, attempts.getLength());
-// NodeList info = dom.getElementsByTagName("jobAttempt");
-// verifyJobAttemptsXML(info, jobsMap.get(id));
-// }
-// }
-//
-// public void verifyJobAttempts(JSONObject info, Job job)
-// throws JSONException {
-//
-// JSONArray attempts = info.getJSONArray("jobAttempt");
-// assertEquals("incorrect number of elements", 2, attempts.length());
-// for (int i = 0; i < attempts.length(); i++) {
-// JSONObject attempt = attempts.getJSONObject(i);
-// verifyJobAttemptsGeneric(job, attempt.getString("nodeHttpAddress"),
-// attempt.getString("nodeId"), attempt.getInt("id"),
-// attempt.getLong("startTime"), attempt.getString("containerId"),
-// attempt.getString("logsLink"));
-// }
-// }
-//
-// public void verifyJobAttemptsXML(NodeList nodes, Job job) {
-//
-// assertEquals("incorrect number of elements", 2, nodes.getLength());
-// for (int i = 0; i < nodes.getLength(); i++) {
-// Element element = (Element) nodes.item(i);
-// verifyJobAttemptsGeneric(job,
-// WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
-// WebServicesTestUtils.getXmlString(element, "nodeId"),
-// WebServicesTestUtils.getXmlInt(element, "id"),
-// WebServicesTestUtils.getXmlLong(element, "startTime"),
-// WebServicesTestUtils.getXmlString(element, "containerId"),
-// WebServicesTestUtils.getXmlString(element, "logsLink"));
-// }
-// }
-//
-// public void verifyJobAttemptsGeneric(Job job, String nodeHttpAddress,
-// String nodeId, int id, long startTime, String containerId, String logsLink) {
-// boolean attemptFound = false;
-// for (AMInfo amInfo : job.getAMInfos()) {
-// if (amInfo.getAppAttemptId().getAttemptId() == id) {
-// attemptFound = true;
-// String nmHost = amInfo.getNodeManagerHost();
-// int nmHttpPort = amInfo.getNodeManagerHttpPort();
-// int nmPort = amInfo.getNodeManagerPort();
-// WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":"
-// + nmHttpPort, nodeHttpAddress);
-// WebServicesTestUtils.checkStringMatch("nodeId",
-// BuilderUtils.newNodeId(nmHost, nmPort).toString(), nodeId);
-// assertTrue("startime not greater than 0", startTime > 0);
-// WebServicesTestUtils.checkStringMatch("containerId", amInfo
-// .getContainerId().toString(), containerId);
-//
-// String localLogsLink =ujoin("node", "containerlogs", containerId,
-// job.getUserName());
-//
-// assertTrue("logsLink", logsLink.contains(localLogsLink));
-// }
-// }
-// assertTrue("attempt: " + id + " was not found", attemptFound);
-// }
-//
-//}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.app2.webapp;
+
+import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.StringReader;
+import java.util.List;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.JobACL;
+import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
+import org.apache.hadoop.mapreduce.v2.app2.AppContext;
+import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
+import org.apache.hadoop.mapreduce.v2.app2.job.Job;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.ClusterInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+/**
+ * Test the app master web service Rest API for getting jobs, a specific job,
+ * and job counters.
+ *
+ * /ws/v1/mapreduce/jobs
+ * /ws/v1/mapreduce/jobs/{jobid}
+ * /ws/v1/mapreduce/jobs/{jobid}/counters
+ * /ws/v1/mapreduce/jobs/{jobid}/jobattempts
+ */
+public class TestAMWebServicesJobs extends JerseyTest {
+
+ private static Configuration conf = new Configuration();
+ private static TestAppContext appContext;
+
+ static class TestAppContext implements AppContext {
+ final ApplicationAttemptId appAttemptID;
+ final ApplicationId appID;
+ final String user = MockJobs.newUserName();
+ final Map<JobId, Job> jobs;
+ final long startTime = System.currentTimeMillis();
+
+ TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
+ appID = MockJobs.newAppID(appid);
+ appAttemptID = MockJobs.newAppAttemptID(appID, 0);
+ jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
+ }
+
+ TestAppContext() {
+ this(0, 1, 2, 1);
+ }
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptID;
+ }
+
+ @Override
+ public ApplicationId getApplicationID() {
+ return appID;
+ }
+
+ @Override
+ public CharSequence getUser() {
+ return user;
+ }
+
+ @Override
+ public Job getJob(JobId jobID) {
+ return jobs.get(jobID);
+ }
+
+ @Override
+ public Map<JobId, Job> getAllJobs() {
+ return jobs; // OK
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public EventHandler getEventHandler() {
+ return null;
+ }
+
+ @Override
+ public Clock getClock() {
+ return null;
+ }
+
+ @Override
+ public String getApplicationName() {
+ return "TestApp";
+ }
+
+ @Override
+ public long getStartTime() {
+ return startTime;
+ }
+
+ @Override
+ public ClusterInfo getClusterInfo() {
+ return null;
+ }
+ }
+
+ private Injector injector = Guice.createInjector(new ServletModule() {
+ @Override
+ protected void configureServlets() {
+
+ appContext = new TestAppContext();
+ bind(JAXBContextResolver.class);
+ bind(AMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ bind(AppContext.class).toInstance(appContext);
+ bind(Configuration.class).toInstance(conf);
+
+ serve("/*").with(GuiceContainer.class);
+ }
+ });
+
+ public class GuiceServletConfig extends GuiceServletContextListener {
+
+ @Override
+ protected Injector getInjector() {
+ return injector;
+ }
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ }
+
+ public TestAMWebServicesJobs() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.mapreduce.v2.app2.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testJobs() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject jobs = json.getJSONObject("jobs");
+ JSONArray arr = jobs.getJSONArray("job");
+ JSONObject info = arr.getJSONObject(0);
+ Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
+ verifyAMJob(info, job);
+
+ }
+
+ @Test
+ public void testJobsSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs/").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject jobs = json.getJSONObject("jobs");
+ JSONArray arr = jobs.getJSONArray("job");
+ JSONObject info = arr.getJSONObject(0);
+ Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
+ verifyAMJob(info, job);
+
+ }
+
+ @Test
+ public void testJobsDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject jobs = json.getJSONObject("jobs");
+ JSONArray arr = jobs.getJSONArray("job");
+ JSONObject info = arr.getJSONObject(0);
+ Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
+ verifyAMJob(info, job);
+
+ }
+
+ @Test
+ public void testJobsXML() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").accept(MediaType.APPLICATION_XML)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList jobs = dom.getElementsByTagName("jobs");
+ assertEquals("incorrect number of elements", 1, jobs.getLength());
+ NodeList job = dom.getElementsByTagName("job");
+ assertEquals("incorrect number of elements", 1, job.getLength());
+ verifyAMJobXML(job, appContext);
+
+ }
+
+ @Test
+ public void testJobId() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("job");
+ verifyAMJob(info, jobsMap.get(id));
+ }
+
+ }
+
+ @Test
+ public void testJobIdSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId + "/").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("job");
+ verifyAMJob(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobIdDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("job");
+ verifyAMJob(info, jobsMap.get(id));
+ }
+
+ }
+
+ @Test
+ public void testJobIdNonExist() throws JSONException, Exception {
+ WebResource r = resource();
+
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs")
+ .path("job_0_1234").get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringMatch("exception message",
+ "java.lang.Exception: job, job_0_1234, is not found", message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "NotFoundException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ }
+ }
+
+ @Test
+ public void testJobIdInvalid() throws JSONException, Exception {
+ WebResource r = resource();
+
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path("job_foo")
+ .accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ verifyJobIdInvalid(message, type, classname);
+ }
+ }
+
+ // verify the exception output default is JSON
+ @Test
+ public void testJobIdInvalidDefault() throws JSONException, Exception {
+ WebResource r = resource();
+
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path("job_foo")
+ .get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ verifyJobIdInvalid(message, type, classname);
+ }
+ }
+
+ // test that the exception output works in XML
+ @Test
+ public void testJobIdInvalidXML() throws JSONException, Exception {
+ WebResource r = resource();
+
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path("job_foo")
+ .accept(MediaType.APPLICATION_XML).get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String msg = response.getEntity(String.class);
+ System.out.println(msg);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(msg));
+ Document dom = db.parse(is);
+ NodeList nodes = dom.getElementsByTagName("RemoteException");
+ Element element = (Element) nodes.item(0);
+ String message = WebServicesTestUtils.getXmlString(element, "message");
+ String type = WebServicesTestUtils.getXmlString(element, "exception");
+ String classname = WebServicesTestUtils.getXmlString(element,
+ "javaClassName");
+ verifyJobIdInvalid(message, type, classname);
+ }
+ }
+
+ private void verifyJobIdInvalid(String message, String type, String classname) {
+ WebServicesTestUtils.checkStringMatch("exception message",
+ "java.lang.Exception: JobId string : job_foo is not properly formed",
+ message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "NotFoundException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ }
+
+ @Test
+ public void testJobIdInvalidBogus() throws JSONException, Exception {
+ WebResource r = resource();
+
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path("bogusfoo")
+ .get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils
+ .checkStringMatch(
+ "exception message",
+ "java.lang.Exception: JobId string : bogusfoo is not properly formed",
+ message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "NotFoundException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ }
+ }
+
+ @Test
+ public void testJobIdXML() throws Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).accept(MediaType.APPLICATION_XML)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList job = dom.getElementsByTagName("job");
+ verifyAMJobXML(job, appContext);
+ }
+
+ }
+
+ public void verifyAMJob(JSONObject info, Job job) throws JSONException {
+
+ assertEquals("incorrect number of elements", 30, info.length());
+
+ // everyone access fields
+ verifyAMJobGeneric(job, info.getString("id"), info.getString("user"),
+ info.getString("name"), info.getString("state"),
+ info.getLong("startTime"), info.getLong("finishTime"),
+ info.getLong("elapsedTime"), info.getInt("mapsTotal"),
+ info.getInt("mapsCompleted"), info.getInt("reducesTotal"),
+ info.getInt("reducesCompleted"),
+ (float) info.getDouble("reduceProgress"),
+ (float) info.getDouble("mapProgress"));
+
+ String diagnostics = "";
+ if (info.has("diagnostics")) {
+ diagnostics = info.getString("diagnostics");
+ }
+
+ // restricted access fields - if security and acls set
+ verifyAMJobGenericSecure(job, info.getInt("mapsPending"),
+ info.getInt("mapsRunning"), info.getInt("reducesPending"),
+ info.getInt("reducesRunning"), info.getBoolean("uberized"),
+ diagnostics, info.getInt("newReduceAttempts"),
+ info.getInt("runningReduceAttempts"),
+ info.getInt("failedReduceAttempts"),
+ info.getInt("killedReduceAttempts"),
+ info.getInt("successfulReduceAttempts"), info.getInt("newMapAttempts"),
+ info.getInt("runningMapAttempts"), info.getInt("failedMapAttempts"),
+ info.getInt("killedMapAttempts"), info.getInt("successfulMapAttempts"));
+
+ Map<JobACL, AccessControlList> allacls = job.getJobACLs();
+ if (allacls != null) {
+
+ for (Map.Entry<JobACL, AccessControlList> entry : allacls.entrySet()) {
+ String expectName = entry.getKey().getAclName();
+ String expectValue = entry.getValue().getAclString();
+ Boolean found = false;
+ // make sure ws includes it
+ if (info.has("acls")) {
+ JSONArray arr = info.getJSONArray("acls");
+
+ for (int i = 0; i < arr.length(); i++) {
+ JSONObject aclInfo = arr.getJSONObject(i);
+ if (expectName.matches(aclInfo.getString("name"))) {
+ found = true;
+ WebServicesTestUtils.checkStringMatch("value", expectValue,
+ aclInfo.getString("value"));
+ }
+ }
+ } else {
+ fail("should have acls in the web service info");
+ }
+ assertTrue("acl: " + expectName + " not found in webservice output",
+ found);
+ }
+ }
+
+ }
+
+ public void verifyAMJobXML(NodeList nodes, TestAppContext appContext) {
+
+ assertEquals("incorrect number of elements", 1, nodes.getLength());
+
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+
+ Job job = appContext.getJob(MRApps.toJobID(WebServicesTestUtils
+ .getXmlString(element, "id")));
+ assertNotNull("Job not found - output incorrect", job);
+
+ verifyAMJobGeneric(job, WebServicesTestUtils.getXmlString(element, "id"),
+ WebServicesTestUtils.getXmlString(element, "user"),
+ WebServicesTestUtils.getXmlString(element, "name"),
+ WebServicesTestUtils.getXmlString(element, "state"),
+ WebServicesTestUtils.getXmlLong(element, "startTime"),
+ WebServicesTestUtils.getXmlLong(element, "finishTime"),
+ WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
+ WebServicesTestUtils.getXmlInt(element, "mapsTotal"),
+ WebServicesTestUtils.getXmlInt(element, "mapsCompleted"),
+ WebServicesTestUtils.getXmlInt(element, "reducesTotal"),
+ WebServicesTestUtils.getXmlInt(element, "reducesCompleted"),
+ WebServicesTestUtils.getXmlFloat(element, "reduceProgress"),
+ WebServicesTestUtils.getXmlFloat(element, "mapProgress"));
+
+ // restricted access fields - if security and acls set
+ verifyAMJobGenericSecure(job,
+ WebServicesTestUtils.getXmlInt(element, "mapsPending"),
+ WebServicesTestUtils.getXmlInt(element, "mapsRunning"),
+ WebServicesTestUtils.getXmlInt(element, "reducesPending"),
+ WebServicesTestUtils.getXmlInt(element, "reducesRunning"),
+ WebServicesTestUtils.getXmlBoolean(element, "uberized"),
+ WebServicesTestUtils.getXmlString(element, "diagnostics"),
+ WebServicesTestUtils.getXmlInt(element, "newReduceAttempts"),
+ WebServicesTestUtils.getXmlInt(element, "runningReduceAttempts"),
+ WebServicesTestUtils.getXmlInt(element, "failedReduceAttempts"),
+ WebServicesTestUtils.getXmlInt(element, "killedReduceAttempts"),
+ WebServicesTestUtils.getXmlInt(element, "successfulReduceAttempts"),
+ WebServicesTestUtils.getXmlInt(element, "newMapAttempts"),
+ WebServicesTestUtils.getXmlInt(element, "runningMapAttempts"),
+ WebServicesTestUtils.getXmlInt(element, "failedMapAttempts"),
+ WebServicesTestUtils.getXmlInt(element, "killedMapAttempts"),
+ WebServicesTestUtils.getXmlInt(element, "successfulMapAttempts"));
+
+ Map<JobACL, AccessControlList> allacls = job.getJobACLs();
+ if (allacls != null) {
+ for (Map.Entry<JobACL, AccessControlList> entry : allacls.entrySet()) {
+ String expectName = entry.getKey().getAclName();
+ String expectValue = entry.getValue().getAclString();
+ Boolean found = false;
+ // make sure ws includes it
+ NodeList id = element.getElementsByTagName("acls");
+ if (id != null) {
+ for (int j = 0; j < id.getLength(); j++) {
+ Element aclElem = (Element) id.item(j);
+ if (aclElem == null) {
+ fail("should have acls in the web service info");
+ }
+ if (expectName.matches(WebServicesTestUtils.getXmlString(aclElem,
+ "name"))) {
+ found = true;
+ WebServicesTestUtils.checkStringMatch("value", expectValue,
+ WebServicesTestUtils.getXmlString(aclElem, "value"));
+ }
+ }
+ } else {
+ fail("should have acls in the web service info");
+ }
+ assertTrue("acl: " + expectName + " not found in webservice output",
+ found);
+ }
+ }
+ }
+ }
+
+ public void verifyAMJobGeneric(Job job, String id, String user, String name,
+ String state, long startTime, long finishTime, long elapsedTime,
+ int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
+ float reduceProgress, float mapProgress) {
+ JobReport report = job.getReport();
+
+ WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
+ id);
+ WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
+ user);
+ WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
+ WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
+ state);
+
+ assertEquals("startTime incorrect", report.getStartTime(), startTime);
+ assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
+ assertEquals("elapsedTime incorrect",
+ Times.elapsed(report.getStartTime(), report.getFinishTime()),
+ elapsedTime);
+ assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
+ assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
+ mapsCompleted);
+ assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
+ assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
+ reducesCompleted);
+ assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
+ mapProgress, 0);
+ assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
+ reduceProgress, 0);
+ }
+
+ public void verifyAMJobGenericSecure(Job job, int mapsPending,
+ int mapsRunning, int reducesPending, int reducesRunning,
+ Boolean uberized, String diagnostics, int newReduceAttempts,
+ int runningReduceAttempts, int failedReduceAttempts,
+ int killedReduceAttempts, int successfulReduceAttempts,
+ int newMapAttempts, int runningMapAttempts, int failedMapAttempts,
+ int killedMapAttempts, int successfulMapAttempts) {
+
+ String diagString = "";
+ List<String> diagList = job.getDiagnostics();
+ if (diagList != null && !diagList.isEmpty()) {
+ StringBuffer b = new StringBuffer();
+ for (String diag : diagList) {
+ b.append(diag);
+ }
+ diagString = b.toString();
+ }
+ WebServicesTestUtils.checkStringMatch("diagnostics", diagString,
+ diagnostics);
+
+ assertEquals("isUber incorrect", job.isUber(), uberized);
+
+ // unfortunately the following fields are all calculated in JobInfo
+ // so not easily accessible without doing all the calculations again.
+ // For now just make sure they are present.
+ assertTrue("mapsPending not >= 0", mapsPending >= 0);
+ assertTrue("mapsRunning not >= 0", mapsRunning >= 0);
+ assertTrue("reducesPending not >= 0", reducesPending >= 0);
+ assertTrue("reducesRunning not >= 0", reducesRunning >= 0);
+
+ assertTrue("newReduceAttempts not >= 0", newReduceAttempts >= 0);
+ assertTrue("runningReduceAttempts not >= 0", runningReduceAttempts >= 0);
+ assertTrue("failedReduceAttempts not >= 0", failedReduceAttempts >= 0);
+ assertTrue("killedReduceAttempts not >= 0", killedReduceAttempts >= 0);
+ assertTrue("successfulReduceAttempts not >= 0",
+ successfulReduceAttempts >= 0);
+
+ assertTrue("newMapAttempts not >= 0", newMapAttempts >= 0);
+ assertTrue("runningMapAttempts not >= 0", runningMapAttempts >= 0);
+ assertTrue("failedMapAttempts not >= 0", failedMapAttempts >= 0);
+ assertTrue("killedMapAttempts not >= 0", killedMapAttempts >= 0);
+ assertTrue("successfulMapAttempts not >= 0", successfulMapAttempts >= 0);
+
+ }
+
+ @Test
+ public void testJobCounters() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("counters")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobCounters");
+ verifyAMJobCounters(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobCountersSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("counters/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobCounters");
+ verifyAMJobCounters(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobCountersDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("counters/").get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobCounters");
+ verifyAMJobCounters(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobCountersXML() throws Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("counters")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList info = dom.getElementsByTagName("jobCounters");
+ verifyAMJobCountersXML(info, jobsMap.get(id));
+ }
+ }
+
+ public void verifyAMJobCounters(JSONObject info, Job job)
+ throws JSONException {
+
+ assertEquals("incorrect number of elements", 2, info.length());
+
+ WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
+ info.getString("id"));
+ // just do simple verification of fields - not data is correct
+ // in the fields
+ JSONArray counterGroups = info.getJSONArray("counterGroup");
+ for (int i = 0; i < counterGroups.length(); i++) {
+ JSONObject counterGroup = counterGroups.getJSONObject(i);
+ String name = counterGroup.getString("counterGroupName");
+ assertTrue("name not set", (name != null && !name.isEmpty()));
+ JSONArray counters = counterGroup.getJSONArray("counter");
+ for (int j = 0; j < counters.length(); j++) {
+ JSONObject counter = counters.getJSONObject(j);
+ String counterName = counter.getString("name");
+ assertTrue("counter name not set",
+ (counterName != null && !counterName.isEmpty()));
+
+ long mapValue = counter.getLong("mapCounterValue");
+ assertTrue("mapCounterValue >= 0", mapValue >= 0);
+
+ long reduceValue = counter.getLong("reduceCounterValue");
+ assertTrue("reduceCounterValue >= 0", reduceValue >= 0);
+
+ long totalValue = counter.getLong("totalCounterValue");
+ assertTrue("totalCounterValue >= 0", totalValue >= 0);
+
+ }
+ }
+ }
+
+ public void verifyAMJobCountersXML(NodeList nodes, Job job) {
+
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+
+ assertNotNull("Job not found - output incorrect", job);
+
+ WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
+ WebServicesTestUtils.getXmlString(element, "id"));
+ // just do simple verification of fields - not data is correct
+ // in the fields
+ NodeList groups = element.getElementsByTagName("counterGroup");
+
+ for (int j = 0; j < groups.getLength(); j++) {
+ Element counters = (Element) groups.item(j);
+ assertNotNull("should have counters in the web service info", counters);
+ String name = WebServicesTestUtils.getXmlString(counters,
+ "counterGroupName");
+ assertTrue("name not set", (name != null && !name.isEmpty()));
+ NodeList counterArr = counters.getElementsByTagName("counter");
+ for (int z = 0; z < counterArr.getLength(); z++) {
+ Element counter = (Element) counterArr.item(z);
+ String counterName = WebServicesTestUtils.getXmlString(counter,
+ "name");
+ assertTrue("counter name not set",
+ (counterName != null && !counterName.isEmpty()));
+
+ long mapValue = WebServicesTestUtils.getXmlLong(counter,
+ "mapCounterValue");
+ assertTrue("mapCounterValue not >= 0", mapValue >= 0);
+
+ long reduceValue = WebServicesTestUtils.getXmlLong(counter,
+ "reduceCounterValue");
+ assertTrue("reduceCounterValue >= 0", reduceValue >= 0);
+
+ long totalValue = WebServicesTestUtils.getXmlLong(counter,
+ "totalCounterValue");
+ assertTrue("totalCounterValue >= 0", totalValue >= 0);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testJobAttempts() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1")
+ .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobAttempts");
+ verifyJobAttempts(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobAttemptsSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1")
+ .path("mapreduce").path("jobs").path(jobId).path("jobattempts/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobAttempts");
+ verifyJobAttempts(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobAttemptsDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1")
+ .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobAttempts");
+ verifyJobAttempts(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobAttemptsXML() throws Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1")
+ .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList attempts = dom.getElementsByTagName("jobAttempts");
+ assertEquals("incorrect number of elements", 1, attempts.getLength());
+ NodeList info = dom.getElementsByTagName("jobAttempt");
+ verifyJobAttemptsXML(info, jobsMap.get(id));
+ }
+ }
+
+ public void verifyJobAttempts(JSONObject info, Job job)
+ throws JSONException {
+
+ JSONArray attempts = info.getJSONArray("jobAttempt");
+ assertEquals("incorrect number of elements", 2, attempts.length());
+ for (int i = 0; i < attempts.length(); i++) {
+ JSONObject attempt = attempts.getJSONObject(i);
+ verifyJobAttemptsGeneric(job, attempt.getString("nodeHttpAddress"),
+ attempt.getString("nodeId"), attempt.getInt("id"),
+ attempt.getLong("startTime"), attempt.getString("containerId"),
+ attempt.getString("logsLink"));
+ }
+ }
+
+ public void verifyJobAttemptsXML(NodeList nodes, Job job) {
+
+ assertEquals("incorrect number of elements", 2, nodes.getLength());
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+ verifyJobAttemptsGeneric(job,
+ WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
+ WebServicesTestUtils.getXmlString(element, "nodeId"),
+ WebServicesTestUtils.getXmlInt(element, "id"),
+ WebServicesTestUtils.getXmlLong(element, "startTime"),
+ WebServicesTestUtils.getXmlString(element, "containerId"),
+ WebServicesTestUtils.getXmlString(element, "logsLink"));
+ }
+ }
+
+ public void verifyJobAttemptsGeneric(Job job, String nodeHttpAddress,
+ String nodeId, int id, long startTime, String containerId, String logsLink) {
+ boolean attemptFound = false;
+ for (AMInfo amInfo : job.getAMInfos()) {
+ if (amInfo.getAppAttemptId().getAttemptId() == id) {
+ attemptFound = true;
+ String nmHost = amInfo.getNodeManagerHost();
+ int nmHttpPort = amInfo.getNodeManagerHttpPort();
+ int nmPort = amInfo.getNodeManagerPort();
+ WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":"
+ + nmHttpPort, nodeHttpAddress);
+ WebServicesTestUtils.checkStringMatch("nodeId",
+ BuilderUtils.newNodeId(nmHost, nmPort).toString(), nodeId);
+ assertTrue("startime not greater than 0", startTime > 0);
+ WebServicesTestUtils.checkStringMatch("containerId", amInfo
+ .getContainerId().toString(), containerId);
+
+ String localLogsLink =ujoin("node", "containerlogs", containerId,
+ job.getUserName());
+
+ assertTrue("logsLink", logsLink.contains(localLogsLink));
+ }
+ }
+ assertTrue("attempt: " + id + " was not found", attemptFound);
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesTasks.java
index 060284d..1637581 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesTasks.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app2/src/test/java/org/apache/hadoop/mapreduce/v2/app2/webapp/TestAMWebServicesTasks.java
@@ -1,830 +1,830 @@
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.hadoop.mapreduce.v2.app2.webapp;
-//
-//import static org.junit.Assert.assertEquals;
-//import static org.junit.Assert.assertNotNull;
-//import static org.junit.Assert.assertTrue;
-//import static org.junit.Assert.fail;
-//
-//import java.io.StringReader;
-//import java.util.Map;
-//
-//import javax.ws.rs.core.MediaType;
-//import javax.xml.parsers.DocumentBuilder;
-//import javax.xml.parsers.DocumentBuilderFactory;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-//import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
-//import org.apache.hadoop.mapreduce.v2.app2.AppContext;
-//import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Job;
-//import org.apache.hadoop.mapreduce.v2.app2.job.Task;
-//import org.apache.hadoop.mapreduce.v2.util.MRApps;
-//import org.apache.hadoop.yarn.Clock;
-//import org.apache.hadoop.yarn.ClusterInfo;
-//import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-//import org.apache.hadoop.yarn.api.records.ApplicationId;
-//import org.apache.hadoop.yarn.event.EventHandler;
-//import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-//import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
-//import org.codehaus.jettison.json.JSONArray;
-//import org.codehaus.jettison.json.JSONException;
-//import org.codehaus.jettison.json.JSONObject;
-//import org.junit.Before;
-//import org.junit.Test;
-//import org.w3c.dom.Document;
-//import org.w3c.dom.Element;
-//import org.w3c.dom.NodeList;
-//import org.xml.sax.InputSource;
-//
-//import com.google.inject.Guice;
-//import com.google.inject.Injector;
-//import com.google.inject.servlet.GuiceServletContextListener;
-//import com.google.inject.servlet.ServletModule;
-//import com.sun.jersey.api.client.ClientResponse;
-//import com.sun.jersey.api.client.ClientResponse.Status;
-//import com.sun.jersey.api.client.UniformInterfaceException;
-//import com.sun.jersey.api.client.WebResource;
-//import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-//import com.sun.jersey.test.framework.JerseyTest;
-//import com.sun.jersey.test.framework.WebAppDescriptor;
-//
-///**
-// * Test the app master web service Rest API for getting tasks, a specific task,
-// * and task counters.
-// *
-// * /ws/v1/mapreduce/jobs/{jobid}/tasks
-// * /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}
-// * /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/counters
-// */
-//public class TestAMWebServicesTasks extends JerseyTest {
-//
-// private static Configuration conf = new Configuration();
-// private static TestAppContext appContext;
-//
-// static class TestAppContext implements AppContext {
-// final ApplicationAttemptId appAttemptID;
-// final ApplicationId appID;
-// final String user = MockJobs.newUserName();
-// final Map<JobId, Job> jobs;
-// final long startTime = System.currentTimeMillis();
-//
-// TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
-// appID = MockJobs.newAppID(appid);
-// appAttemptID = MockJobs.newAppAttemptID(appID, 0);
-// jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
-// }
-//
-// TestAppContext() {
-// this(0, 1, 2, 1);
-// }
-//
-// @Override
-// public ApplicationAttemptId getApplicationAttemptId() {
-// return appAttemptID;
-// }
-//
-// @Override
-// public ApplicationId getApplicationID() {
-// return appID;
-// }
-//
-// @Override
-// public CharSequence getUser() {
-// return user;
-// }
-//
-// @Override
-// public Job getJob(JobId jobID) {
-// return jobs.get(jobID);
-// }
-//
-// @Override
-// public Map<JobId, Job> getAllJobs() {
-// return jobs; // OK
-// }
-//
-// @SuppressWarnings("rawtypes")
-// @Override
-// public EventHandler getEventHandler() {
-// return null;
-// }
-//
-// @Override
-// public Clock getClock() {
-// return null;
-// }
-//
-// @Override
-// public String getApplicationName() {
-// return "TestApp";
-// }
-//
-// @Override
-// public long getStartTime() {
-// return startTime;
-// }
-//
-// @Override
-// public ClusterInfo getClusterInfo() {
-// return null;
-// }
-// }
-//
-// private Injector injector = Guice.createInjector(new ServletModule() {
-// @Override
-// protected void configureServlets() {
-//
-// appContext = new TestAppContext();
-// bind(JAXBContextResolver.class);
-// bind(AMWebServices.class);
-// bind(GenericExceptionHandler.class);
-// bind(AppContext.class).toInstance(appContext);
-// bind(Configuration.class).toInstance(conf);
-//
-// serve("/*").with(GuiceContainer.class);
-// }
-// });
-//
-// public class GuiceServletConfig extends GuiceServletContextListener {
-//
-// @Override
-// protected Injector getInjector() {
-// return injector;
-// }
-// }
-//
-// @Before
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// }
-//
-// public TestAMWebServicesTasks() {
-// super(new WebAppDescriptor.Builder(
-// "org.apache.hadoop.mapreduce.v2.app2.webapp")
-// .contextListenerClass(GuiceServletConfig.class)
-// .filterClass(com.google.inject.servlet.GuiceFilter.class)
-// .contextPath("jersey-guice-filter").servletPath("/").build());
-// }
-//
-// @Test
-// public void testTasks() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject tasks = json.getJSONObject("tasks");
-// JSONArray arr = tasks.getJSONArray("task");
-// assertEquals("incorrect number of elements", 2, arr.length());
-//
-// verifyAMTask(arr, jobsMap.get(id), null);
-// }
-// }
-//
-// @Test
-// public void testTasksDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject tasks = json.getJSONObject("tasks");
-// JSONArray arr = tasks.getJSONArray("task");
-// assertEquals("incorrect number of elements", 2, arr.length());
-//
-// verifyAMTask(arr, jobsMap.get(id), null);
-// }
-// }
-//
-// @Test
-// public void testTasksSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks/")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject tasks = json.getJSONObject("tasks");
-// JSONArray arr = tasks.getJSONArray("task");
-// assertEquals("incorrect number of elements", 2, arr.length());
-//
-// verifyAMTask(arr, jobsMap.get(id), null);
-// }
-// }
-//
-// @Test
-// public void testTasksXML() throws JSONException, Exception {
-//
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks")
-// .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList tasks = dom.getElementsByTagName("tasks");
-// assertEquals("incorrect number of elements", 1, tasks.getLength());
-// NodeList task = dom.getElementsByTagName("task");
-// verifyAMTaskXML(task, jobsMap.get(id));
-// }
-// }
-//
-// @Test
-// public void testTasksQueryMap() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// String type = "m";
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").queryParam("type", type)
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject tasks = json.getJSONObject("tasks");
-// JSONArray arr = tasks.getJSONArray("task");
-// assertEquals("incorrect number of elements", 1, arr.length());
-// verifyAMTask(arr, jobsMap.get(id), type);
-// }
-// }
-//
-// @Test
-// public void testTasksQueryReduce() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// String type = "r";
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").queryParam("type", type)
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject tasks = json.getJSONObject("tasks");
-// JSONArray arr = tasks.getJSONArray("task");
-// assertEquals("incorrect number of elements", 1, arr.length());
-// verifyAMTask(arr, jobsMap.get(id), type);
-// }
-// }
-//
-// @Test
-// public void testTasksQueryInvalid() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// // tasktype must be exactly either "m" or "r"
-// String tasktype = "reduce";
-//
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
-// .path("tasks").queryParam("type", tasktype)
-// .accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// WebServicesTestUtils.checkStringMatch("exception message",
-// "java.lang.Exception: tasktype must be either m or r", message);
-// WebServicesTestUtils.checkStringMatch("exception type",
-// "BadRequestException", type);
-// WebServicesTestUtils.checkStringMatch("exception classname",
-// "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskId() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid)
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("task");
-// verifyAMSingleTask(info, task);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskIdSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid + "/")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("task");
-// verifyAMSingleTask(info, task);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskIdDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid)
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("task");
-// verifyAMSingleTask(info, task);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskIdBogus() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// String tid = "bogustaskid";
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
-// .path("tasks").path(tid).get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// WebServicesTestUtils.checkStringMatch("exception message",
-// "java.lang.Exception: TaskId string : "
-// + "bogustaskid is not properly formed", message);
-// WebServicesTestUtils.checkStringMatch("exception type",
-// "NotFoundException", type);
-// WebServicesTestUtils.checkStringMatch("exception classname",
-// "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskIdNonExist() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// String tid = "task_0_0000_m_000000";
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
-// .path("tasks").path(tid).get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// WebServicesTestUtils.checkStringMatch("exception message",
-// "java.lang.Exception: task not found with id task_0_0000_m_000000",
-// message);
-// WebServicesTestUtils.checkStringMatch("exception type",
-// "NotFoundException", type);
-// WebServicesTestUtils.checkStringMatch("exception classname",
-// "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskIdInvalid() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// String tid = "task_0_0000_d_000000";
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
-// .path("tasks").path(tid).get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// WebServicesTestUtils.checkStringMatch("exception message",
-// "java.lang.Exception: Bad TaskType identifier. TaskId string : "
-// + "task_0_0000_d_000000 is not properly formed.", message);
-// WebServicesTestUtils.checkStringMatch("exception type",
-// "NotFoundException", type);
-// WebServicesTestUtils.checkStringMatch("exception classname",
-// "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskIdInvalid2() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// String tid = "task_0_m_000000";
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
-// .path("tasks").path(tid).get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// WebServicesTestUtils.checkStringMatch("exception message",
-// "java.lang.Exception: TaskId string : "
-// + "task_0_m_000000 is not properly formed", message);
-// WebServicesTestUtils.checkStringMatch("exception type",
-// "NotFoundException", type);
-// WebServicesTestUtils.checkStringMatch("exception classname",
-// "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskIdInvalid3() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// String tid = "task_0_0000_m";
-// try {
-// r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
-// .path("tasks").path(tid).get(JSONObject.class);
-// fail("should have thrown exception on invalid uri");
-// } catch (UniformInterfaceException ue) {
-// ClientResponse response = ue.getResponse();
-// assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject msg = response.getEntity(JSONObject.class);
-// JSONObject exception = msg.getJSONObject("RemoteException");
-// assertEquals("incorrect number of elements", 3, exception.length());
-// String message = exception.getString("message");
-// String type = exception.getString("exception");
-// String classname = exception.getString("javaClassName");
-// WebServicesTestUtils.checkStringMatch("exception message",
-// "java.lang.Exception: TaskId string : "
-// + "task_0_0000_m is not properly formed", message);
-// WebServicesTestUtils.checkStringMatch("exception type",
-// "NotFoundException", type);
-// WebServicesTestUtils.checkStringMatch("exception classname",
-// "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskIdXML() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid)
-// .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-//
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList nodes = dom.getElementsByTagName("task");
-// for (int i = 0; i < nodes.getLength(); i++) {
-// Element element = (Element) nodes.item(i);
-// verifyAMSingleTaskXML(element, task);
-// }
-// }
-// }
-// }
-//
-// public void verifyAMSingleTask(JSONObject info, Task task)
-// throws JSONException {
-// assertEquals("incorrect number of elements", 8, info.length());
-//
-// verifyTaskGeneric(task, info.getString("id"), info.getString("state"),
-// info.getString("type"), info.getString("successfulAttempt"),
-// info.getLong("startTime"), info.getLong("finishTime"),
-// info.getLong("elapsedTime"), (float) info.getDouble("progress"));
-// }
-//
-// public void verifyAMTask(JSONArray arr, Job job, String type)
-// throws JSONException {
-// for (Task task : job.getTasks().values()) {
-// TaskId id = task.getID();
-// String tid = MRApps.toString(id);
-// Boolean found = false;
-// if (type != null && task.getType() == MRApps.taskType(type)) {
-//
-// for (int i = 0; i < arr.length(); i++) {
-// JSONObject info = arr.getJSONObject(i);
-// if (tid.matches(info.getString("id"))) {
-// found = true;
-// verifyAMSingleTask(info, task);
-// }
-// }
-// assertTrue("task with id: " + tid + " not in web service output", found);
-// }
-// }
-// }
-//
-// public void verifyTaskGeneric(Task task, String id, String state,
-// String type, String successfulAttempt, long startTime, long finishTime,
-// long elapsedTime, float progress) {
-//
-// TaskId taskid = task.getID();
-// String tid = MRApps.toString(taskid);
-// TaskReport report = task.getReport();
-//
-// WebServicesTestUtils.checkStringMatch("id", tid, id);
-// WebServicesTestUtils.checkStringMatch("type", task.getType().toString(),
-// type);
-// WebServicesTestUtils.checkStringMatch("state", report.getTaskState()
-// .toString(), state);
-// // not easily checked without duplicating logic, just make sure its here
-// assertNotNull("successfulAttempt null", successfulAttempt);
-// assertEquals("startTime wrong", report.getStartTime(), startTime);
-// assertEquals("finishTime wrong", report.getFinishTime(), finishTime);
-// assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
-// assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f);
-// }
-//
-// public void verifyAMSingleTaskXML(Element element, Task task) {
-// verifyTaskGeneric(task, WebServicesTestUtils.getXmlString(element, "id"),
-// WebServicesTestUtils.getXmlString(element, "state"),
-// WebServicesTestUtils.getXmlString(element, "type"),
-// WebServicesTestUtils.getXmlString(element, "successfulAttempt"),
-// WebServicesTestUtils.getXmlLong(element, "startTime"),
-// WebServicesTestUtils.getXmlLong(element, "finishTime"),
-// WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
-// WebServicesTestUtils.getXmlFloat(element, "progress"));
-// }
-//
-// public void verifyAMTaskXML(NodeList nodes, Job job) {
-//
-// assertEquals("incorrect number of elements", 2, nodes.getLength());
-//
-// for (Task task : job.getTasks().values()) {
-// TaskId id = task.getID();
-// String tid = MRApps.toString(id);
-// Boolean found = false;
-// for (int i = 0; i < nodes.getLength(); i++) {
-// Element element = (Element) nodes.item(i);
-//
-// if (tid.matches(WebServicesTestUtils.getXmlString(element, "id"))) {
-// found = true;
-// verifyAMSingleTaskXML(element, task);
-// }
-// }
-// assertTrue("task with id: " + tid + " not in web service output", found);
-// }
-// }
-//
-// @Test
-// public void testTaskIdCounters() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid).path("counters")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("jobTaskCounters");
-// verifyAMJobTaskCounters(info, task);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskIdCountersSlash() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid).path("counters/")
-// .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("jobTaskCounters");
-// verifyAMJobTaskCounters(info, task);
-// }
-// }
-// }
-//
-// @Test
-// public void testTaskIdCountersDefault() throws JSONException, Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid).path("counters")
-// .get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
-// JSONObject json = response.getEntity(JSONObject.class);
-// assertEquals("incorrect number of elements", 1, json.length());
-// JSONObject info = json.getJSONObject("jobTaskCounters");
-// verifyAMJobTaskCounters(info, task);
-// }
-// }
-// }
-//
-// @Test
-// public void testJobTaskCountersXML() throws Exception {
-// WebResource r = resource();
-// Map<JobId, Job> jobsMap = appContext.getAllJobs();
-// for (JobId id : jobsMap.keySet()) {
-// String jobId = MRApps.toString(id);
-// for (Task task : jobsMap.get(id).getTasks().values()) {
-//
-// String tid = MRApps.toString(task.getID());
-// ClientResponse response = r.path("ws").path("v1").path("mapreduce")
-// .path("jobs").path(jobId).path("tasks").path(tid).path("counters")
-// .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-// assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
-// String xml = response.getEntity(String.class);
-// DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-// DocumentBuilder db = dbf.newDocumentBuilder();
-// InputSource is = new InputSource();
-// is.setCharacterStream(new StringReader(xml));
-// Document dom = db.parse(is);
-// NodeList info = dom.getElementsByTagName("jobTaskCounters");
-// verifyAMTaskCountersXML(info, task);
-// }
-// }
-// }
-//
-// public void verifyAMJobTaskCounters(JSONObject info, Task task)
-// throws JSONException {
-//
-// assertEquals("incorrect number of elements", 2, info.length());
-//
-// WebServicesTestUtils.checkStringMatch("id", MRApps.toString(task.getID()),
-// info.getString("id"));
-// // just do simple verification of fields - not data is correct
-// // in the fields
-// JSONArray counterGroups = info.getJSONArray("taskCounterGroup");
-// for (int i = 0; i < counterGroups.length(); i++) {
-// JSONObject counterGroup = counterGroups.getJSONObject(i);
-// String name = counterGroup.getString("counterGroupName");
-// assertTrue("name not set", (name != null && !name.isEmpty()));
-// JSONArray counters = counterGroup.getJSONArray("counter");
-// for (int j = 0; j < counters.length(); j++) {
-// JSONObject counter = counters.getJSONObject(j);
-// String counterName = counter.getString("name");
-// assertTrue("name not set",
-// (counterName != null && !counterName.isEmpty()));
-// long value = counter.getLong("value");
-// assertTrue("value >= 0", value >= 0);
-// }
-// }
-// }
-//
-// public void verifyAMTaskCountersXML(NodeList nodes, Task task) {
-//
-// for (int i = 0; i < nodes.getLength(); i++) {
-//
-// Element element = (Element) nodes.item(i);
-// WebServicesTestUtils.checkStringMatch("id",
-// MRApps.toString(task.getID()),
-// WebServicesTestUtils.getXmlString(element, "id"));
-// // just do simple verification of fields - not data is correct
-// // in the fields
-// NodeList groups = element.getElementsByTagName("taskCounterGroup");
-//
-// for (int j = 0; j < groups.getLength(); j++) {
-// Element counters = (Element) groups.item(j);
-// assertNotNull("should have counters in the web service info", counters);
-// String name = WebServicesTestUtils.getXmlString(counters,
-// "counterGroupName");
-// assertTrue("name not set", (name != null && !name.isEmpty()));
-// NodeList counterArr = counters.getElementsByTagName("counter");
-// for (int z = 0; z < counterArr.getLength(); z++) {
-// Element counter = (Element) counterArr.item(z);
-// String counterName = WebServicesTestUtils.getXmlString(counter,
-// "name");
-// assertTrue("counter name not set",
-// (counterName != null && !counterName.isEmpty()));
-//
-// long value = WebServicesTestUtils.getXmlLong(counter, "value");
-// assertTrue("value not >= 0", value >= 0);
-//
-// }
-// }
-// }
-// }
-//
-//}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.app2.webapp;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.StringReader;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
+import org.apache.hadoop.mapreduce.v2.app2.AppContext;
+import org.apache.hadoop.mapreduce.v2.app2.MockJobs;
+import org.apache.hadoop.mapreduce.v2.app2.job.Job;
+import org.apache.hadoop.mapreduce.v2.app2.job.Task;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.ClusterInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+/**
+ * Test the app master web service Rest API for getting tasks, a specific task,
+ * and task counters.
+ *
+ * /ws/v1/mapreduce/jobs/{jobid}/tasks
+ * /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}
+ * /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/counters
+ */
+public class TestAMWebServicesTasks extends JerseyTest {
+
+ private static Configuration conf = new Configuration();
+ private static TestAppContext appContext;
+
+ static class TestAppContext implements AppContext {
+ final ApplicationAttemptId appAttemptID;
+ final ApplicationId appID;
+ final String user = MockJobs.newUserName();
+ final Map<JobId, Job> jobs;
+ final long startTime = System.currentTimeMillis();
+
+ TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
+ appID = MockJobs.newAppID(appid);
+ appAttemptID = MockJobs.newAppAttemptID(appID, 0);
+ jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
+ }
+
+ TestAppContext() {
+ this(0, 1, 2, 1);
+ }
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptID;
+ }
+
+ @Override
+ public ApplicationId getApplicationID() {
+ return appID;
+ }
+
+ @Override
+ public CharSequence getUser() {
+ return user;
+ }
+
+ @Override
+ public Job getJob(JobId jobID) {
+ return jobs.get(jobID);
+ }
+
+ @Override
+ public Map<JobId, Job> getAllJobs() {
+ return jobs; // OK
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public EventHandler getEventHandler() {
+ return null;
+ }
+
+ @Override
+ public Clock getClock() {
+ return null;
+ }
+
+ @Override
+ public String getApplicationName() {
+ return "TestApp";
+ }
+
+ @Override
+ public long getStartTime() {
+ return startTime;
+ }
+
+ @Override
+ public ClusterInfo getClusterInfo() {
+ return null;
+ }
+ }
+
+ private Injector injector = Guice.createInjector(new ServletModule() {
+ @Override
+ protected void configureServlets() {
+
+ appContext = new TestAppContext();
+ bind(JAXBContextResolver.class);
+ bind(AMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ bind(AppContext.class).toInstance(appContext);
+ bind(Configuration.class).toInstance(conf);
+
+ serve("/*").with(GuiceContainer.class);
+ }
+ });
+
+ public class GuiceServletConfig extends GuiceServletContextListener {
+
+ @Override
+ protected Injector getInjector() {
+ return injector;
+ }
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ }
+
+ public TestAMWebServicesTasks() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.mapreduce.v2.app2.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testTasks() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject tasks = json.getJSONObject("tasks");
+ JSONArray arr = tasks.getJSONArray("task");
+ assertEquals("incorrect number of elements", 2, arr.length());
+
+ verifyAMTask(arr, jobsMap.get(id), null);
+ }
+ }
+
+ @Test
+ public void testTasksDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject tasks = json.getJSONObject("tasks");
+ JSONArray arr = tasks.getJSONArray("task");
+ assertEquals("incorrect number of elements", 2, arr.length());
+
+ verifyAMTask(arr, jobsMap.get(id), null);
+ }
+ }
+
+ @Test
+ public void testTasksSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject tasks = json.getJSONObject("tasks");
+ JSONArray arr = tasks.getJSONArray("task");
+ assertEquals("incorrect number of elements", 2, arr.length());
+
+ verifyAMTask(arr, jobsMap.get(id), null);
+ }
+ }
+
+ @Test
+ public void testTasksXML() throws JSONException, Exception {
+
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList tasks = dom.getElementsByTagName("tasks");
+ assertEquals("incorrect number of elements", 1, tasks.getLength());
+ NodeList task = dom.getElementsByTagName("task");
+ verifyAMTaskXML(task, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testTasksQueryMap() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ String type = "m";
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").queryParam("type", type)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject tasks = json.getJSONObject("tasks");
+ JSONArray arr = tasks.getJSONArray("task");
+ assertEquals("incorrect number of elements", 1, arr.length());
+ verifyAMTask(arr, jobsMap.get(id), type);
+ }
+ }
+
+ @Test
+ public void testTasksQueryReduce() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ String type = "r";
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").queryParam("type", type)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject tasks = json.getJSONObject("tasks");
+ JSONArray arr = tasks.getJSONArray("task");
+ assertEquals("incorrect number of elements", 1, arr.length());
+ verifyAMTask(arr, jobsMap.get(id), type);
+ }
+ }
+
+ @Test
+ public void testTasksQueryInvalid() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ // tasktype must be exactly either "m" or "r"
+ String tasktype = "reduce";
+
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
+ .path("tasks").queryParam("type", tasktype)
+ .accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringMatch("exception message",
+ "java.lang.Exception: tasktype must be either m or r", message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "BadRequestException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskId() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("task");
+ verifyAMSingleTask(info, task);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskIdSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid + "/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("task");
+ verifyAMSingleTask(info, task);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskIdDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("task");
+ verifyAMSingleTask(info, task);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskIdBogus() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ String tid = "bogustaskid";
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
+ .path("tasks").path(tid).get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringMatch("exception message",
+ "java.lang.Exception: TaskId string : "
+ + "bogustaskid is not properly formed", message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "NotFoundException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskIdNonExist() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ String tid = "task_0_0000_m_000000";
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
+ .path("tasks").path(tid).get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringMatch("exception message",
+ "java.lang.Exception: task not found with id task_0_0000_m_000000",
+ message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "NotFoundException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskIdInvalid() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ String tid = "task_0_0000_d_000000";
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
+ .path("tasks").path(tid).get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringMatch("exception message",
+ "java.lang.Exception: Bad TaskType identifier. TaskId string : "
+ + "task_0_0000_d_000000 is not properly formed.", message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "NotFoundException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskIdInvalid2() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ String tid = "task_0_m_000000";
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
+ .path("tasks").path(tid).get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringMatch("exception message",
+ "java.lang.Exception: TaskId string : "
+ + "task_0_m_000000 is not properly formed", message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "NotFoundException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskIdInvalid3() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ String tid = "task_0_0000_m";
+ try {
+ r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId)
+ .path("tasks").path(tid).get(JSONObject.class);
+ fail("should have thrown exception on invalid uri");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringMatch("exception message",
+ "java.lang.Exception: TaskId string : "
+ + "task_0_0000_m is not properly formed", message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "NotFoundException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskIdXML() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid)
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList nodes = dom.getElementsByTagName("task");
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+ verifyAMSingleTaskXML(element, task);
+ }
+ }
+ }
+ }
+
+ public void verifyAMSingleTask(JSONObject info, Task task)
+ throws JSONException {
+ assertEquals("incorrect number of elements", 8, info.length());
+
+ verifyTaskGeneric(task, info.getString("id"), info.getString("state"),
+ info.getString("type"), info.getString("successfulAttempt"),
+ info.getLong("startTime"), info.getLong("finishTime"),
+ info.getLong("elapsedTime"), (float) info.getDouble("progress"));
+ }
+
+ public void verifyAMTask(JSONArray arr, Job job, String type)
+ throws JSONException {
+ for (Task task : job.getTasks().values()) {
+ TaskId id = task.getID();
+ String tid = MRApps.toString(id);
+ Boolean found = false;
+ if (type != null && task.getType() == MRApps.taskType(type)) {
+
+ for (int i = 0; i < arr.length(); i++) {
+ JSONObject info = arr.getJSONObject(i);
+ if (tid.matches(info.getString("id"))) {
+ found = true;
+ verifyAMSingleTask(info, task);
+ }
+ }
+ assertTrue("task with id: " + tid + " not in web service output", found);
+ }
+ }
+ }
+
+ public void verifyTaskGeneric(Task task, String id, String state,
+ String type, String successfulAttempt, long startTime, long finishTime,
+ long elapsedTime, float progress) {
+
+ TaskId taskid = task.getID();
+ String tid = MRApps.toString(taskid);
+ TaskReport report = task.getReport();
+
+ WebServicesTestUtils.checkStringMatch("id", tid, id);
+ WebServicesTestUtils.checkStringMatch("type", task.getType().toString(),
+ type);
+ WebServicesTestUtils.checkStringMatch("state", report.getTaskState()
+ .toString(), state);
+ // not easily checked without duplicating logic, just make sure its here
+ assertNotNull("successfulAttempt null", successfulAttempt);
+ assertEquals("startTime wrong", report.getStartTime(), startTime);
+ assertEquals("finishTime wrong", report.getFinishTime(), finishTime);
+ assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
+ assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f);
+ }
+
+ public void verifyAMSingleTaskXML(Element element, Task task) {
+ verifyTaskGeneric(task, WebServicesTestUtils.getXmlString(element, "id"),
+ WebServicesTestUtils.getXmlString(element, "state"),
+ WebServicesTestUtils.getXmlString(element, "type"),
+ WebServicesTestUtils.getXmlString(element, "successfulAttempt"),
+ WebServicesTestUtils.getXmlLong(element, "startTime"),
+ WebServicesTestUtils.getXmlLong(element, "finishTime"),
+ WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
+ WebServicesTestUtils.getXmlFloat(element, "progress"));
+ }
+
+ public void verifyAMTaskXML(NodeList nodes, Job job) {
+
+ assertEquals("incorrect number of elements", 2, nodes.getLength());
+
+ for (Task task : job.getTasks().values()) {
+ TaskId id = task.getID();
+ String tid = MRApps.toString(id);
+ Boolean found = false;
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+
+ if (tid.matches(WebServicesTestUtils.getXmlString(element, "id"))) {
+ found = true;
+ verifyAMSingleTaskXML(element, task);
+ }
+ }
+ assertTrue("task with id: " + tid + " not in web service output", found);
+ }
+ }
+
+ @Test
+ public void testTaskIdCounters() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid).path("counters")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobTaskCounters");
+ verifyAMJobTaskCounters(info, task);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskIdCountersSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid).path("counters/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobTaskCounters");
+ verifyAMJobTaskCounters(info, task);
+ }
+ }
+ }
+
+ @Test
+ public void testTaskIdCountersDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid).path("counters")
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobTaskCounters");
+ verifyAMJobTaskCounters(info, task);
+ }
+ }
+ }
+
+ @Test
+ public void testJobTaskCountersXML() throws Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+ for (Task task : jobsMap.get(id).getTasks().values()) {
+
+ String tid = MRApps.toString(task.getID());
+ ClientResponse response = r.path("ws").path("v1").path("mapreduce")
+ .path("jobs").path(jobId).path("tasks").path(tid).path("counters")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList info = dom.getElementsByTagName("jobTaskCounters");
+ verifyAMTaskCountersXML(info, task);
+ }
+ }
+ }
+
+ public void verifyAMJobTaskCounters(JSONObject info, Task task)
+ throws JSONException {
+
+ assertEquals("incorrect number of elements", 2, info.length());
+
+ WebServicesTestUtils.checkStringMatch("id", MRApps.toString(task.getID()),
+ info.getString("id"));
+ // just do simple verification of fields - not data is correct
+ // in the fields
+ JSONArray counterGroups = info.getJSONArray("taskCounterGroup");
+ for (int i = 0; i < counterGroups.length(); i++) {
+ JSONObject counterGroup = counterGroups.getJSONObject(i);
+ String name = counterGroup.getString("counterGroupName");
+ assertTrue("name not set", (name != null && !name.isEmpty()));
+ JSONArray counters = counterGroup.getJSONArray("counter");
+ for (int j = 0; j < counters.length(); j++) {
+ JSONObject counter = counters.getJSONObject(j);
+ String counterName = counter.getString("name");
+ assertTrue("name not set",
+ (counterName != null && !counterName.isEmpty()));
+ long value = counter.getLong("value");
+ assertTrue("value >= 0", value >= 0);
+ }
+ }
+ }
+
+ public void verifyAMTaskCountersXML(NodeList nodes, Task task) {
+
+ for (int i = 0; i < nodes.getLength(); i++) {
+
+ Element element = (Element) nodes.item(i);
+ WebServicesTestUtils.checkStringMatch("id",
+ MRApps.toString(task.getID()),
+ WebServicesTestUtils.getXmlString(element, "id"));
+ // just do simple verification of fields - not data is correct
+ // in the fields
+ NodeList groups = element.getElementsByTagName("taskCounterGroup");
+
+ for (int j = 0; j < groups.getLength(); j++) {
+ Element counters = (Element) groups.item(j);
+ assertNotNull("should have counters in the web service info", counters);
+ String name = WebServicesTestUtils.getXmlString(counters,
+ "counterGroupName");
+ assertTrue("name not set", (name != null && !name.isEmpty()));
+ NodeList counterArr = counters.getElementsByTagName("counter");
+ for (int z = 0; z < counterArr.getLength(); z++) {
+ Element counter = (Element) counterArr.item(z);
+ String counterName = WebServicesTestUtils.getXmlString(counter,
+ "name");
+ assertTrue("counter name not set",
+ (counterName != null && !counterName.isEmpty()));
+
+ long value = WebServicesTestUtils.getXmlLong(counter, "value");
+ assertTrue("value not >= 0", value >= 0);
+
+ }
+ }
+ }
+ }
+
+}
