[REEF-1831] Setting FileSytemUrl for YarnConfiguration used in FileSystem
* Inject FileSystemUrl in YarnConfigurationConstructor
* Set FileSystemUrl for FileSystem in YarnConfigurationCOnstructor
* Bind FileSystemUrl to driver configuration
JIRA: [REEF-1831](https://issues.apache.org/jira/browse/REEF-1831)
This closes #1332
diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java
index f536bd3..fcf03ba 100644
--- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java
+++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/YarnJobSubmissionClient.java
@@ -277,7 +277,7 @@
if (!yarnSubmission.getFileSystemUrl().equalsIgnoreCase(FileSystemUrl.DEFAULT_VALUE)) {
LOG.log(Level.INFO, "getFileSystemUrl: {0}", yarnSubmission.getFileSystemUrl());
} else {
- LOG.log(Level.INFO, "FileSystemUrl is not set");
+ LOG.log(Level.INFO, "FileSystemUrl is not set, use default from the environment.");
}
final List<String> launchCommandPrefix = new ArrayList<String>() {{
@@ -288,6 +288,7 @@
.bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
.bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class)
.bindNamedParameter(JobSubmissionDirectoryPrefix.class, yarnSubmission.getJobSubmissionDirectoryPrefix())
+ .bindNamedParameter(FileSystemUrl.class, yarnSubmission.getFileSystemUrl())
.bindList(DriverLaunchCommandPrefix.class, launchCommandPrefix)
.build();
final YarnJobSubmissionClient client = Tang.Factory.getTang().newInjector(yarnJobSubmissionClientConfig)
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
index f89afa0..de8d0b1 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
@@ -43,7 +43,6 @@
import org.apache.reef.runtime.common.driver.resourcemanager.RuntimeStatusEventImpl;
import org.apache.reef.runtime.common.files.REEFFileNames;
import org.apache.reef.runtime.yarn.client.unmanaged.YarnProxyUser;
-import org.apache.reef.runtime.yarn.driver.parameters.FileSystemUrl;
import org.apache.reef.runtime.yarn.driver.parameters.JobSubmissionDirectory;
import org.apache.reef.runtime.yarn.driver.parameters.YarnHeartbeatPeriod;
import org.apache.reef.tang.InjectionFuture;
@@ -88,9 +87,7 @@
private final DriverStatusManager driverStatusManager;
private final String trackingUrl;
private final String amRegistrationHost;
-
private final String jobSubmissionDirectory;
- private final String fileSystemUrl;
private final REEFFileNames reefFileNames;
private final RackNameFormatter rackNameFormatter;
private final InjectionFuture<ProgressProvider> progressProvider;
@@ -99,7 +96,6 @@
private YarnContainerManager(
@Parameter(YarnHeartbeatPeriod.class) final int yarnRMHeartbeatPeriod,
@Parameter(JobSubmissionDirectory.class) final String jobSubmissionDirectory,
- @Parameter(FileSystemUrl.class) final String fileSystemUrl,
final YarnConfiguration yarnConf,
final YarnProxyUser yarnProxyUser,
final REEFEventHandlers reefEventHandlers,
@@ -130,14 +126,11 @@
this.nodeManager = new NMClientAsyncImpl(this);
this.jobSubmissionDirectory = jobSubmissionDirectory;
- this.fileSystemUrl = fileSystemUrl;
this.reefFileNames = reefFileNames;
this.progressProvider = progressProvider;
- LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {2}, fileSystemUrl: {3}, " +
- "jobSubmissionDirectory: {4}.",
- new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.fileSystemUrl,
- this.jobSubmissionDirectory});
+ LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {3}, jobSubmissionDirectory: {4}.",
+ new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.jobSubmissionDirectory});
}
/**
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java
index 1f318a7..a54b8e3 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/util/YarnConfigurationConstructor.java
@@ -19,20 +19,35 @@
package org.apache.reef.runtime.yarn.util;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.reef.runtime.yarn.driver.parameters.FileSystemUrl;
import org.apache.reef.tang.ExternalConstructor;
+import org.apache.reef.tang.annotations.Parameter;
import javax.inject.Inject;
+import java.util.logging.Level;
+import java.util.logging.Logger;
/**
* An external constructor that creates YarnConfiguration instances.
*/
public final class YarnConfigurationConstructor implements ExternalConstructor<YarnConfiguration> {
+
+ private static final Logger LOG = Logger.getLogger(YarnConfigurationConstructor.class.getName());
+
+ private final String fileSystemUrl;
+
@Inject
- YarnConfigurationConstructor() {
+ YarnConfigurationConstructor(@Parameter(FileSystemUrl.class) final String fileSystemUrl) {
+ this.fileSystemUrl = fileSystemUrl;
}
@Override
public YarnConfiguration newInstance() {
- return new YarnConfiguration();
+ YarnConfiguration yarnConfiguration = new YarnConfiguration();
+ if (!fileSystemUrl.equals(FileSystemUrl.DEFAULT_VALUE)) {
+ yarnConfiguration.set("fs.defaultFS", fileSystemUrl);
+ LOG.log(Level.INFO, "Set fileSystemUrl in YarnConfigurationConstructor: {0}", fileSystemUrl);
+ }
+ return yarnConfiguration;
}
}
