Fixes #11 Apps are failing to run on cluster (#12)
* Created new property so that 'fs.defaultFS' is set in hadoop
config that is passed to YarnTwillRunnerService
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
index dcf2041..2c76fec 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
@@ -82,6 +82,7 @@
yarnConfig = new YarnConfiguration();
yarnConfig.set("mapreduce.framework.name", "yarn");
yarnConfig.set("yarn.resourcemanager.hostname", getYarnResourceManager());
+ yarnConfig.set("fs.defaultFS", getDfsRoot());
}
return yarnConfig;
}
@@ -96,6 +97,10 @@
FluoYarnProperties.ZOOKEEPERS_DEFAULT);
}
+ public String getDfsRoot() {
+ return props.getProperty(FluoYarnProperties.DFS_ROOT_PROP, FluoYarnProperties.DFS_ROOT_DEFAULT);
+ }
+
public int getWorkerCores() {
return Integer.valueOf(props.getProperty(FluoYarnProperties.WORKER_NUM_CORES_PROP,
FluoYarnProperties.WORKER_NUM_CORES_DEFAULT));
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
index 4d34647..d61dca6 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
@@ -26,8 +26,10 @@
// General properties
public static final String YARN_RESOURCE_MANAGER_PROP = YARN_PREFIX + ".resource.manager";
public static final String ZOOKEEPERS_PROP = YARN_PREFIX + ".zookeepers";
+ public static final String DFS_ROOT_PROP = YARN_PREFIX + ".dfs.root";
public static final String YARN_RESOURCE_MANAGER_DEFAULT = "localhost";
public static final String ZOOKEEPERS_DEFAULT = "localhost/fluo-yarn";
+ public static final String DFS_ROOT_DEFAULT = "hdfs://localhost:8020/";
// Worker properties
public static final String WORKER_INSTANCES_PROP = YARN_PREFIX + ".worker.instances";
diff --git a/distribution/conf/fluo-yarn.properties b/distribution/conf/fluo-yarn.properties
index 996b132..0af7bf8 100644
--- a/distribution/conf/fluo-yarn.properties
+++ b/distribution/conf/fluo-yarn.properties
@@ -25,15 +25,21 @@
## Zookeeper connection string specifying host and chroot where Fluo YARN launcher stores data.
## A chroot directory suffix (with max depth of 1) must be specified but doesn't need to be named '/fluo-yarn'.
#fluo.yarn.zookeepers=localhost/fluo-yarn
+
## YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property in Hadoop's yarn-site.xml
#fluo.yarn.resource.manager=localhost
+## HDFS root path. Should be prefixed with 'fs.defaultFS' property in Hadoop's core-site.xml
+#fluo.yarn.dfs.root=hdfs://localhost:8020/
+
## Oracle properties
## -----------------
## Number of oracle yarn instances
#fluo.yarn.oracle.instances=1
+
## Max memory of Oracle yarn containers (in MB)
#fluo.yarn.oracle.max.memory.mb=512
+
## Number of Oracle virtual cores
#fluo.yarn.oracle.num.cores=1
@@ -41,11 +47,13 @@
## -----------------
## Number of Worker instances
#fluo.yarn.worker.instances=1
+
## Max memory of worker YARN containers (in MB). If YARN is killing worker processes consider
## increasing twill.java.reserved.memory.mb (which defaults to 200 and is set in yarn-site.xml).
## The twill.java.reserved.memory.mb config determines the gap between the YARN memory limit set
## below and the java -Xmx setting. For example, if max memory is 1024 and twill reserved memory
## is 200, the java -Xmx setting will be 1024-200 = 824 MB.
#fluo.yarn.worker.max.memory.mb=1024
+
## Number of worker virtual cores
#fluo.yarn.worker.num.cores=1
