[REEF-1482] Driver does not exit even if all the task exit normally
Currently, when all the tasks and all the evaluators are completed,
sometimes driver still doen't shut down and hungs there forever.
This happens intermittently. When there are many nodes like 500 nodes
in IMRU runs in Yarn cluster, the issue can happen in every 2 or 3 runs.
The investigation shows there is a potential dead lock in ResourceManagerStatus.
This PR is to resolve this issue by reducing the scope of code under locks.
With the fixes, I have tested 10 times with 500 nodes in cluster, there is no repro any more.
JIRA:
[REEF-1482](https://issues.apache.org/jira/browse/REEF-1482)
Pull request:
This closes #1162
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/idle/DriverIdleManager.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/idle/DriverIdleManager.java
index 2792790..13019fd 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/idle/DriverIdleManager.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/idle/DriverIdleManager.java
@@ -77,6 +77,8 @@
isIdle &= idleMessage.isIdle();
}
+ LOG.log(IDLE_REASONS_LEVEL, "onPotentiallyIdle: isIdle: " + isIdle);
+
if (isIdle) {
LOG.log(Level.INFO, "All components indicated idle. Initiating Driver shutdown.");
driverStatusManagerImpl.onComplete();
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceManagerStatus.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceManagerStatus.java
index 4b19330..54e2450 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceManagerStatus.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceManagerStatus.java
@@ -71,16 +71,18 @@
}
@Override
- public synchronized void onNext(final RuntimeStatusEvent runtimeStatusEvent) {
+ public void onNext(final RuntimeStatusEvent runtimeStatusEvent) {
final State newState = runtimeStatusEvent.getState();
LOG.log(Level.FINEST, "Runtime status: {0}", runtimeStatusEvent);
- this.outstandingContainerRequests = runtimeStatusEvent.getOutstandingContainerRequests().orElse(0);
- this.containerAllocationCount = runtimeStatusEvent.getContainerAllocationList().size();
+ synchronized(this) {
+ this.outstandingContainerRequests = runtimeStatusEvent.getOutstandingContainerRequests().orElse(0);
+ this.containerAllocationCount = runtimeStatusEvent.getContainerAllocationList().size();
- this.setState(newState);
+ this.setState(newState);
+ }
switch (newState) {
case FAILED:
@@ -147,7 +149,7 @@
this.driverStatusManager.onComplete();
}
- private synchronized void onRMRunning(final RuntimeStatusEvent runtimeStatusEvent) {
+ private void onRMRunning(final RuntimeStatusEvent runtimeStatusEvent) {
assert runtimeStatusEvent.getState() == State.RUNNING;
if (this.isIdle()) {
this.driverIdleManager.get().onPotentiallyIdle(IDLE_MESSAGE);
