YARN-6616: YARN AHS shows submitTime for jobs same as startTime. Contributed by Prabhu Joseph
(cherry picked from commit 04105bbfdb041a41062c856632641140de84fba8)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
index d572d62..3fd0af0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
@@ -62,6 +62,22 @@
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken) {
+ return newInstance(applicationId, applicationAttemptId, user, queue, name,
+ host, rpcPort, clientToAMToken, state, diagnostics, url,
+ startTime, startTime, launchTime, finishTime, finalStatus, appResources,
+ origTrackingUrl, progress, applicationType, amRmToken);
+ }
+
+ @Private
+ @Unstable
+ public static ApplicationReport newInstance(ApplicationId applicationId,
+ ApplicationAttemptId applicationAttemptId, String user, String queue,
+ String name, String host, int rpcPort, Token clientToAMToken,
+ YarnApplicationState state, String diagnostics, String url,
+ long startTime, long submitTime, long launchTime, long finishTime,
+ FinalApplicationStatus finalStatus,
+ ApplicationResourceUsageReport appResources, String origTrackingUrl,
+ float progress, String applicationType, Token amRmToken) {
ApplicationReport report = Records.newRecord(ApplicationReport.class);
report.setApplicationId(applicationId);
report.setCurrentApplicationAttemptId(applicationAttemptId);
@@ -75,6 +91,7 @@
report.setDiagnostics(diagnostics);
report.setTrackingUrl(url);
report.setStartTime(startTime);
+ report.setSubmitTime(submitTime);
report.setLaunchTime(launchTime);
report.setFinishTime(finishTime);
report.setFinalApplicationStatus(finalStatus);
@@ -102,7 +119,7 @@
ApplicationReport report =
newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url,
- startTime, 0, finishTime, finalStatus, appResources,
+ startTime, 0, 0, finishTime, finalStatus, appResources,
origTrackingUrl, progress, applicationType, amRmToken);
report.setApplicationTags(tags);
report.setUnmanagedApp(unmanagedApplication);
@@ -124,10 +141,30 @@
float progress, String applicationType, Token amRmToken, Set<String> tags,
boolean unmanagedApplication, Priority priority,
String appNodeLabelExpression, String amNodeLabelExpression) {
+ return newInstance(applicationId, applicationAttemptId, user, queue, name,
+ host, rpcPort, clientToAMToken, state, diagnostics, url,
+ startTime, startTime, launchTime, finishTime, finalStatus, appResources,
+ origTrackingUrl, progress, applicationType, amRmToken, tags,
+ unmanagedApplication, priority, appNodeLabelExpression,
+ amNodeLabelExpression);
+ }
+
+ @Private
+ @Unstable
+ public static ApplicationReport newInstance(ApplicationId applicationId,
+ ApplicationAttemptId applicationAttemptId, String user, String queue,
+ String name, String host, int rpcPort, Token clientToAMToken,
+ YarnApplicationState state, String diagnostics, String url,
+ long startTime, long submitTime, long launchTime, long finishTime,
+ FinalApplicationStatus finalStatus,
+ ApplicationResourceUsageReport appResources, String origTrackingUrl,
+ float progress, String applicationType, Token amRmToken, Set<String> tags,
+ boolean unmanagedApplication, Priority priority,
+ String appNodeLabelExpression, String amNodeLabelExpression) {
ApplicationReport report =
newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
- launchTime, finishTime, finalStatus, appResources,
+ submitTime, launchTime, finishTime, finalStatus, appResources,
origTrackingUrl, progress, applicationType, amRmToken);
report.setApplicationTags(tags);
report.setUnmanagedApp(unmanagedApplication);
@@ -311,6 +348,14 @@
@Unstable
public abstract void setStartTime(long startTime);
+ @Public
+ @Stable
+ public abstract long getSubmitTime();
+
+ @Private
+ @Unstable
+ public abstract void setSubmitTime(long submitTime);
+
@Private
@Unstable
public abstract void setLaunchTime(long setLaunchTime);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 7cc85de..51efd15 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -247,6 +247,7 @@
optional string amNodeLabelExpression = 25;
repeated AppTimeoutsMapProto appTimeouts = 26;
optional int64 launchTime = 27;
+ optional int64 submitTime = 28;
}
message AppTimeoutsMapProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
index f0e3ca2..e05e57e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
@@ -102,6 +102,8 @@
Assert.assertEquals(report, expectedReports.get(0));
Assert.assertEquals(report.getApplicationId().toString(), expectedReports
.get(0).getApplicationId().toString());
+ Assert.assertEquals(report.getSubmitTime(), expectedReports.get(0)
+ .getSubmitTime());
client.stop();
}
@@ -332,7 +334,7 @@
ApplicationReport.newInstance(applicationId,
ApplicationAttemptId.newInstance(applicationId, 1), "user",
"queue", "appname", "host", 124, null,
- YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0,
+ YarnApplicationState.RUNNING, "diagnostics", "url", 1, 2, 3, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
null);
List<ApplicationReport> applicationReports =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
index ddd9acc..5209021 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
@@ -221,6 +221,12 @@
}
@Override
+ public long getSubmitTime() {
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getSubmitTime();
+ }
+
+ @Override
public long getLaunchTime() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
return p.getLaunchTime();
@@ -431,6 +437,12 @@
}
@Override
+ public void setSubmitTime(long submitTime) {
+ maybeInitBuilder();
+ builder.setSubmitTime(submitTime);
+ }
+
+ @Override
public void setFinalApplicationStatus(FinalApplicationStatus finishState) {
maybeInitBuilder();
if (finishState == null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index e9d2c5e..625a5b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -137,9 +137,9 @@
currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(),
appHistory.getApplicationName(), host, rpcPort, null,
appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
- trackingUrl, appHistory.getStartTime(), 0, appHistory.getFinishTime(),
- appHistory.getFinalApplicationStatus(), null, "", 100,
- appHistory.getApplicationType(), null);
+ trackingUrl, appHistory.getStartTime(), appHistory.getSubmitTime(), 0,
+ appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(),
+ null, "", 100, appHistory.getApplicationType(), null);
}
private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index b4c91f9..ca78f06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -249,6 +249,7 @@
String type = null;
boolean unmanagedApplication = false;
long createdTime = 0;
+ long submittedTime = 0;
long finishedTime = 0;
float progress = 0.0f;
int applicationPriority = 0;
@@ -280,10 +281,11 @@
return new ApplicationReportExt(ApplicationReport.newInstance(
ApplicationId.fromString(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null,
- state, diagnosticsInfo, null, createdTime, finishedTime,
- finalStatus, null, null, progress, type, null, appTags,
- unmanagedApplication, Priority.newInstance(applicationPriority),
- appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
+ state, diagnosticsInfo, null, createdTime, submittedTime, 0,
+ finishedTime, finalStatus, null, null, progress, type, null,
+ appTags, unmanagedApplication, Priority.newInstance(
+ applicationPriority), appNodeLabelExpression,
+ amNodeLabelExpression), appViewACLs);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue =
@@ -328,6 +330,8 @@
entityInfo.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)
.toString();
}
+ submittedTime = parseLong(entityInfo,
+ ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO);
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds = parseLong(entityInfo,
@@ -438,10 +442,10 @@
return new ApplicationReportExt(ApplicationReport.newInstance(
ApplicationId.fromString(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
- diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
- appResources, null, progress, type, null, appTags, unmanagedApplication,
- Priority.newInstance(applicationPriority), appNodeLabelExpression,
- amNodeLabelExpression), appViewACLs);
+ diagnosticsInfo, null, createdTime, submittedTime, 0, finishedTime,
+ finalStatus, appResources, null, progress, type, null, appTags,
+ unmanagedApplication, Priority.newInstance(applicationPriority),
+ appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
}
private static long parseLong(Map<String, Object> entityInfo,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index ecaaf1e..96c8131 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -214,6 +214,7 @@
}
Assert.assertEquals(Integer.MAX_VALUE + 2L
+ app.getApplicationId().getId(), app.getStartTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime());
Assert.assertEquals(Integer.MAX_VALUE + 3L
+ +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
index 84ddd99..1dd0dad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
@@ -401,6 +401,7 @@
assertEquals("test app", app.get("name"));
assertEquals(round == 0 ? "test diagnostics info" : "",
app.get("diagnosticsInfo"));
+ assertEquals(Integer.MAX_VALUE + 1L, app.get("submittedTime"));
assertEquals("test queue", app.get("queue"));
assertEquals("user1", app.get("user"));
assertEquals("test app type", app.get("type"));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
index b4687a4..c4de022 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
@@ -53,7 +53,7 @@
protected String originalTrackingUrl;
protected String trackingUrl;
protected FinalApplicationStatus finalAppStatus;
- protected long submittedTime;
+ private long submittedTime;
protected long startedTime;
private long launchTime;
protected long finishedTime;
@@ -87,7 +87,7 @@
diagnosticsInfo = app.getDiagnostics();
trackingUrl = app.getTrackingUrl();
originalTrackingUrl = app.getOriginalTrackingUrl();
- submittedTime = app.getStartTime();
+ submittedTime = app.getSubmitTime();
startedTime = app.getStartTime();
launchTime = app.getLaunchTime();
finishedTime = app.getFinishTime();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index c6d68b3..9d8151a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -263,6 +263,7 @@
final String name = newAppName();
final String queue = newQueue();
final long start = 123456 + i * 1000;
+ final long submit = start + i * 50;
final long launch = start + i * 100;
final long finish = 234567 + i * 1000;
final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE;
@@ -300,6 +301,11 @@
}
@Override
+ public long getSubmitTime() {
+ return submit;
+ }
+
+ @Override
public long getLaunchTime() {
return launch;
}
@@ -353,9 +359,9 @@
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
- getTrackingUrl(), getLaunchTime(), getStartTime(), getFinishTime(),
- getFinalApplicationStatus(), usageReport , null, getProgress(),
- type, null);
+ getTrackingUrl(), getStartTime(), getSubmitTime(), getLaunchTime(),
+ getFinishTime(), getFinalApplicationStatus(), usageReport, null,
+ getProgress(), type, null);
return report;
}
