YARN-7371. Added allocateRequestId in NMContainerStatus for recovery. Contributed by Chandni Singh
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index 19a5177..cf32880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -131,12 +131,16 @@
return fs;
}
+ protected MiniYARNCluster getYarnCluster() {
+ return yarnCluster;
+ }
+
protected void setupInternal(int numNodeManager)
throws Exception {
LOG.info("Starting up YARN cluster");
-// Logger rootLogger = LogManager.getRootLogger();
-// rootLogger.setLevel(Level.DEBUG);
- setConf(new YarnConfiguration());
+ if (conf == null) {
+ setConf(new YarnConfiguration());
+ }
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
// reduce the teardown waiting time
conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 64fcf57..f98d90a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -18,12 +18,13 @@
package org.apache.hadoop.yarn.service;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.Component;
@@ -41,17 +42,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
+import java.util.*;
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
/**
* End to end tests to test deploying services with MiniYarnCluster and a in-JVM
@@ -67,7 +63,8 @@
@Before
public void setup() throws Exception {
- setupInternal(NUM_NMS);
+ File tmpYarnDir = new File("target", "tmp");
+ FileUtils.deleteQuietly(tmpYarnDir);
}
@After
@@ -84,6 +81,7 @@
// 6. Destroy the service
@Test (timeout = 200000)
public void testCreateFlexStopDestroyService() throws Exception {
+ setupInternal(NUM_NMS);
ServiceClient client = createClient();
Service exampleApp = createExampleApplication();
client.actionCreate(exampleApp);
@@ -135,6 +133,7 @@
// Check containers for compa started before containers for compb
@Test (timeout = 200000)
public void testComponentStartOrder() throws Exception {
+ setupInternal(NUM_NMS);
ServiceClient client = createClient();
Service exampleApp = new Service();
exampleApp.setName("teststartorder");
@@ -155,6 +154,70 @@
client.actionDestroy(exampleApp.getName());
}
+ // Test to verify recovery of SeviceMaster after RM is restarted.
+ // 1. Create an example service.
+ // 2. Restart RM.
+ // 3. Fail the application attempt.
+ // 4. Verify ServiceMaster recovers.
+ @Test(timeout = 200000)
+ public void testRecoverComponentsAfterRMRestart() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+ conf.setBoolean(
+ YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
+ conf.setLong(YarnConfiguration.NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ 500L);
+
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
+ setConf(conf);
+ setupInternal(NUM_NMS);
+
+ ServiceClient client = createClient();
+ Service exampleApp = createExampleApplication();
+ client.actionCreate(exampleApp);
+ waitForAllCompToBeReady(client, exampleApp);
+
+ LOG.info("Restart the resource manager");
+ getYarnCluster().restartResourceManager(
+ getYarnCluster().getActiveRMIndex());
+ GenericTestUtils.waitFor(() ->
+ getYarnCluster().getResourceManager().getServiceState() ==
+ org.apache.hadoop.service.Service.STATE.STARTED, 2000, 200000);
+ Assert.assertTrue("node managers connected",
+ getYarnCluster().waitForNodeManagersToConnect(5000));
+
+ ApplicationId exampleAppId = ApplicationId.fromString(exampleApp.getId());
+ ApplicationAttemptId applicationAttemptId = client.getYarnClient()
+ .getApplicationReport(exampleAppId).getCurrentApplicationAttemptId();
+
+ Multimap<String, String> containersBeforeFailure = getContainersForAllComp(
+ client, exampleApp);
+
+ LOG.info("Fail the application attempt {}", applicationAttemptId);
+ client.getYarnClient().failApplicationAttempt(applicationAttemptId);
+ //wait until attempt 2 is running
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ApplicationReport ar = client.getYarnClient()
+ .getApplicationReport(exampleAppId);
+ return ar.getCurrentApplicationAttemptId().getAttemptId() == 2 &&
+ ar.getYarnApplicationState() == YarnApplicationState.RUNNING;
+ } catch (YarnException | IOException e) {
+ throw new RuntimeException("while waiting", e);
+ }
+ }, 2000, 200000);
+
+ Multimap<String, String> containersAfterFailure = getContainersForAllComp(
+ client, exampleApp);
+ Assert.assertEquals("component container affected by restart",
+ containersBeforeFailure, containersAfterFailure);
+
+ LOG.info("Stop/destroy service {}", exampleApp);
+ client.actionStop(exampleApp.getName(), true);
+ client.actionDestroy(exampleApp.getName());
+ }
+
// Check containers launched are in dependency order
// Get all containers into a list and sort based on container launch time e.g.
// compa-c1, compa-c2, compb-c1, compb-c2;
@@ -297,6 +360,24 @@
}, 2000, 200000);
}
+ /**
+ * Get all containers of a service.
+ */
+ private Multimap<String, String> getContainersForAllComp(ServiceClient client,
+ Service example) throws IOException, YarnException {
+
+ Multimap<String, String> allContainers = HashMultimap.create();
+ Service retrievedApp = client.getStatus(example.getName());
+ retrievedApp.getComponents().forEach(component -> {
+ if (component.getContainers() != null) {
+ component.getContainers().forEach(container -> {
+ allContainers.put(component.getName(), container.getId());
+ });
+ }
+ });
+ return allContainers;
+ }
+
private ServiceClient createClient() throws Exception {
ServiceClient client = new ServiceClient() {
@Override protected Path addJarResource(String appName,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index 07917ef..9e7d132 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -115,7 +115,7 @@
this(containerID, 0, hostName, appSubmitter, r, expiryTimeStamp,
masterKeyId, rmIdentifier, priority, creationTime,
logAggregationContext, nodeLabelExpression, containerType,
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, -1);
}
public ContainerTokenIdentifier(ContainerId containerID, int containerVersion,
@@ -123,6 +123,19 @@
int masterKeyId, long rmIdentifier, Priority priority, long creationTime,
LogAggregationContext logAggregationContext, String nodeLabelExpression,
ContainerType containerType, ExecutionType executionType) {
+
+ this(containerID, containerVersion, hostName, appSubmitter, r,
+ expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime,
+ logAggregationContext, nodeLabelExpression, containerType,
+ executionType, -1);
+ }
+
+ public ContainerTokenIdentifier(ContainerId containerID, int containerVersion,
+ String hostName, String appSubmitter, Resource r, long expiryTimeStamp,
+ int masterKeyId, long rmIdentifier, Priority priority, long creationTime,
+ LogAggregationContext logAggregationContext, String nodeLabelExpression,
+ ContainerType containerType, ExecutionType executionType,
+ long allocationRequestId) {
ContainerTokenIdentifierProto.Builder builder =
ContainerTokenIdentifierProto.newBuilder();
if (containerID != null) {
@@ -152,6 +165,7 @@
}
builder.setContainerType(convertToProtoFormat(containerType));
builder.setExecutionType(convertToProtoFormat(executionType));
+ builder.setAllocationRequestId(allocationRequestId);
proto = builder.build();
}
@@ -243,6 +257,10 @@
return new LogAggregationContextPBImpl(proto.getLogAggregationContext());
}
+ public long getAllocationRequestId() {
+ return proto.getAllocationRequestId();
+ }
+
@Override
public void write(DataOutput out) throws IOException {
LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
index 851920d..d8288ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
@@ -53,6 +53,7 @@
optional ContainerTypeProto containerType = 12;
optional ExecutionTypeProto executionType = 13 [default = GUARANTEED];
optional int32 version = 14 [default = 0];
+ optional int64 allocation_request_id = 15 [default = -1];
}
message ClientToAMTokenIdentifierProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
index 180add8..1a095f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
@@ -41,14 +41,14 @@
long creationTime) {
return newInstance(containerId, version, containerState, allocatedResource,
diagnostics, containerExitStatus, priority, creationTime,
- CommonNodeLabelsManager.NO_LABEL, ExecutionType.GUARANTEED);
+ CommonNodeLabelsManager.NO_LABEL, ExecutionType.GUARANTEED, -1);
}
public static NMContainerStatus newInstance(ContainerId containerId,
int version, ContainerState containerState, Resource allocatedResource,
String diagnostics, int containerExitStatus, Priority priority,
long creationTime, String nodeLabelExpression,
- ExecutionType executionType) {
+ ExecutionType executionType, long allocationRequestId) {
NMContainerStatus status =
Records.newRecord(NMContainerStatus.class);
status.setContainerId(containerId);
@@ -61,6 +61,7 @@
status.setCreationTime(creationTime);
status.setNodeLabelExpression(nodeLabelExpression);
status.setExecutionType(executionType);
+ status.setAllocationRequestId(allocationRequestId);
return status;
}
@@ -130,6 +131,19 @@
public abstract void setNodeLabelExpression(
String nodeLabelExpression);
+ /**
+ * @return the <em>ID</em> corresponding to the original allocation request.
+ */
+ public abstract long getAllocationRequestId();
+
+ /**
+ * Set the <em>ID</em> corresponding to the original allocation request.
+ *
+ * @param allocationRequestId the <em>ID</em> corresponding to the original
+ * allocation request.
+ */
+ public abstract void setAllocationRequestId(long allocationRequestId);
+
public int getVersion() {
return 0;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
index 38df5f6..8ed02fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
@@ -91,7 +91,8 @@
.append("Diagnostics: ").append(getDiagnostics()).append(", ")
.append("ExitStatus: ").append(getContainerExitStatus()).append(", ")
.append("NodeLabelExpression: ").append(getNodeLabelExpression())
- .append("Priority: ").append(getPriority())
+ .append("Priority: ").append(getPriority()).append(", ")
+ .append("AllocationRequestId: ").append(getAllocationRequestId())
.append("]");
return sb.toString();
}
@@ -270,6 +271,18 @@
builder.setExecutionType(convertToProtoFormat(executionType));
}
+ @Override
+ public long getAllocationRequestId() {
+ NMContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getAllocationRequestId());
+ }
+
+ @Override
+ public void setAllocationRequestId(long allocationRequestId) {
+ maybeInitBuilder();
+ builder.setAllocationRequestId(allocationRequestId);
+ }
+
private void mergeLocalToBuilder() {
if (this.containerId != null
&& !((ContainerIdPBImpl) containerId).getProto().equals(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
index ede4958..1f53648 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
@@ -564,7 +564,7 @@
tokenSecretManager.getCurrentKey().getKeyId(), rmIdentifier,
schedulerKey.getPriority(), currTime,
null, CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK,
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, schedulerKey.getAllocationRequestId());
byte[] pwd =
tokenSecretManager.createPassword(containerTokenIdentifier);
Token containerToken = newContainerToken(node.getNodeId(), pwd,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 8e59f14..8c4fc69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -176,6 +176,7 @@
optional string nodeLabelExpression = 8;
optional int32 version = 9;
optional ExecutionTypeProto executionType = 10 [default = GUARANTEED];
+ optional int64 allocation_request_id = 11 [default = -1];
}
message SCMUploaderNotifyRequestProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 55119e0..831ba0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -437,7 +437,7 @@
throws IOException {
StartContainerRequest req = rcs.getStartRequest();
ContainerLaunchContext launchContext = req.getContainerLaunchContext();
- ContainerTokenIdentifier token = null;
+ ContainerTokenIdentifier token;
if(rcs.getCapability() != null) {
ContainerTokenIdentifier originalToken =
BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
@@ -449,7 +449,8 @@
originalToken.getCreationTime(),
originalToken.getLogAggregationContext(),
originalToken.getNodeLabelExpression(),
- originalToken.getContainerType(), originalToken.getExecutionType());
+ originalToken.getContainerType(), originalToken.getExecutionType(),
+ originalToken.getAllocationRequestId());
} else {
token = BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 17b24b4..901b013 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -844,12 +844,14 @@
public NMContainerStatus getNMContainerStatus() {
this.readLock.lock();
try {
- return NMContainerStatus.newInstance(this.containerId, this.version,
- getCurrentState(), getResource(), diagnostics.toString(), exitCode,
+ return NMContainerStatus.newInstance(this.containerId,
+ this.version, getCurrentState(), getResource(),
+ diagnostics.toString(), exitCode,
containerTokenIdentifier.getPriority(),
containerTokenIdentifier.getCreationTime(),
containerTokenIdentifier.getNodeLabelExpression(),
- containerTokenIdentifier.getExecutionType());
+ containerTokenIdentifier.getExecutionType(),
+ containerTokenIdentifier.getAllocationRequestId());
} finally {
this.readLock.unlock();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 8ce6eb8..53f43e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -571,6 +571,7 @@
status.getPriority(), null);
container.setVersion(status.getVersion());
container.setExecutionType(status.getExecutionType());
+ container.setAllocationRequestId(status.getAllocationRequestId());
ApplicationAttemptId attemptId =
container.getId().getApplicationAttemptId();
RMContainer rmContainer = new RMContainerImpl(container,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index db63cd8..ce71afa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -655,7 +655,8 @@
container.getNodeId(), getUser(), container.getResource(),
container.getPriority(), rmContainer.getCreationTime(),
this.logAggregationContext, rmContainer.getNodeLabelExpression(),
- containerType, container.getExecutionType()));
+ containerType, container.getExecutionType(),
+ container.getAllocationRequestId()));
updateNMToken(container);
} catch (IllegalArgumentException e) {
// DNS might be down, skip returning this container.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
index 677aa14..191900b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
@@ -167,7 +167,7 @@
}
/**
- * Helper function for creating ContainerTokens
+ * Helper function for creating ContainerTokens.
*
* @param containerId Container Id
* @param containerVersion Container Version
@@ -183,37 +183,13 @@
Resource capability, Priority priority, long createTime) {
return createContainerToken(containerId, containerVersion, nodeId,
appSubmitter, capability, priority, createTime,
- null, null, ContainerType.TASK);
+ null, null, ContainerType.TASK,
+ ExecutionType.GUARANTEED, -1);
}
/**
* Helper function for creating ContainerTokens.
*
- * @param containerId containerId.
- * @param containerVersion containerVersion.
- * @param nodeId nodeId.
- * @param appSubmitter appSubmitter.
- * @param capability capability.
- * @param priority priority.
- * @param createTime createTime.
- * @param logAggregationContext logAggregationContext.
- * @param nodeLabelExpression nodeLabelExpression.
- * @param containerType containerType.
- * @return the container-token.
- */
- public Token createContainerToken(ContainerId containerId,
- int containerVersion, NodeId nodeId, String appSubmitter,
- Resource capability, Priority priority, long createTime,
- LogAggregationContext logAggregationContext, String nodeLabelExpression,
- ContainerType containerType) {
- return createContainerToken(containerId, containerVersion, nodeId,
- appSubmitter, capability, priority, createTime, null, null,
- ContainerType.TASK, ExecutionType.GUARANTEED);
- }
-
- /**
- * Helper function for creating ContainerTokens
- *
* @param containerId Container Id
* @param containerVersion Container version
* @param nodeId Node Id
@@ -225,13 +201,15 @@
* @param nodeLabelExpression Node Label Expression
* @param containerType Container Type
* @param execType Execution Type
+ * @param allocationRequestId allocationRequestId
* @return the container-token
*/
public Token createContainerToken(ContainerId containerId,
int containerVersion, NodeId nodeId, String appSubmitter,
Resource capability, Priority priority, long createTime,
LogAggregationContext logAggregationContext, String nodeLabelExpression,
- ContainerType containerType, ExecutionType execType) {
+ ContainerType containerType, ExecutionType execType,
+ long allocationRequestId) {
byte[] password;
ContainerTokenIdentifier tokenIdentifier;
long expiryTimeStamp =
@@ -246,7 +224,7 @@
this.currentMasterKey.getMasterKey().getKeyId(),
ResourceManager.getClusterTimeStamp(), priority, createTime,
logAggregationContext, nodeLabelExpression, containerType,
- execType);
+ execType, allocationRequestId);
password = this.createPassword(tokenIdentifier);
} finally {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index f0f51f3..a4f18ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -2113,7 +2113,7 @@
NMContainerStatus.newInstance(containerId, 0, containerState,
Resource.newInstance(1024, 1), "recover container", 0,
Priority.newInstance(0), 0, nodeLabelExpression,
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, -1);
return containerReport;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 2bd745b..1cb2d0df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -2088,17 +2088,17 @@
NMContainerStatus.newInstance(c1, 1, ContainerState.RUNNING,
Resource.newInstance(1024, 1), "Dummy Queued OC",
ContainerExitStatus.INVALID, Priority.newInstance(5), 1234, "",
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, -1);
NMContainerStatus runningOpp =
NMContainerStatus.newInstance(c2, 1, ContainerState.RUNNING,
Resource.newInstance(2048, 1), "Dummy Running OC",
ContainerExitStatus.INVALID, Priority.newInstance(6), 1234, "",
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, -1);
NMContainerStatus runningGuar =
NMContainerStatus.newInstance(c3, 1, ContainerState.RUNNING,
Resource.newInstance(2048, 1), "Dummy Running GC",
ContainerExitStatus.INVALID, Priority.newInstance(6), 1234, "",
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, -1);
req.setContainerStatuses(Arrays.asList(queuedOpp, runningOpp, runningGuar));
// trying to register a invalid node.
RegisterNodeManagerResponse response =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index b1ca72a..6f54d47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -295,12 +295,13 @@
int containerVersion, NodeId nodeId, String appSubmitter,
Resource capability, Priority priority, long createTime,
LogAggregationContext logAggregationContext, String nodeLabelExp,
- ContainerType containerType, ExecutionType executionType) {
+ ContainerType containerType, ExecutionType executionType,
+ long allocationRequestId) {
numRetries++;
return super.createContainerToken(containerId, containerVersion,
nodeId, appSubmitter, capability, priority, createTime,
logAggregationContext, nodeLabelExp, containerType,
- executionType);
+ executionType, allocationRequestId);
}
};
}
