[GEARPUMP-254] fix gearpump on Yarn
Be sure to do all of the following to help us incorporate your contribution
quickly and easily:
- [x] Make sure the commit message is formatted like:
`[GEARPUMP-<Jira issue #>] Meaningful description of pull request`
- [x] Make sure tests pass via `sbt clean test`.
- [x] Make sure old documentation affected by the pull request has been updated and new documentation added for new functionality.
Author: huafengw <fvunicorn@gmail.com>
Closes #128 from huafengw/fixYarn.
diff --git a/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/appmaster/Command.scala b/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/appmaster/Command.scala
index 711506a..fb482c4 100644
--- a/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/appmaster/Command.scala
+++ b/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/appmaster/Command.scala
@@ -48,7 +48,7 @@
: String = {
val exe = config.getString(java)
- s"$exe -cp ${classPath.mkString(":")}:" +
+ s"$exe -noverify -cp ${classPath.mkString(":")}:" +
"$CLASSPATH " + properties.mkString(" ") +
s" $mainClazz ${cliOpts.mkString(" ")} 2>&1 | /usr/bin/tee -a ${LOG_DIR_EXPANSION_VAR}/stderr"
}
diff --git a/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/client/LaunchCluster.scala b/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/client/LaunchCluster.scala
index 98cff61..267d588 100644
--- a/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/client/LaunchCluster.scala
+++ b/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/client/LaunchCluster.scala
@@ -185,8 +185,8 @@
if (parsed.getBoolean(VERBOSE)) {
LogUtil.verboseLogToConsole()
}
- val userName = inputAkkaConf.getString(CONTAINER_USER)
- if (userName != null) {
+ if (inputAkkaConf.hasPath(CONTAINER_USER)) {
+ val userName = inputAkkaConf.getString(CONTAINER_USER)
System.setProperty("HADOOP_USER_NAME", userName)
}
diff --git a/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/appmaster/CommandSpec.scala b/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/appmaster/CommandSpec.scala
deleted file mode 100644
index c4c5a65..0000000
--- a/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/appmaster/CommandSpec.scala
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gearpump.experiments.yarn.appmaster
-
-import com.typesafe.config.ConfigFactory
-import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
-
-import org.apache.gearpump.cluster.TestUtil
-import org.apache.gearpump.transport.HostPort
-
-class CommandSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
- val config = ConfigFactory.parseString(
-
- """
- |
- |gearpump {
- | yarn {
- | client {
- | package -path = "/user/gearpump/gearpump.zip"
- | }
- |
- | applicationmaster {
- | ## Memory of YarnAppMaster
- | command = "$JAVA_HOME/bin/java -Xmx512m"
- | memory = "512"
- | vcores = "1"
- | queue = "default"
- | }
- |
- | master {
- | ## Memory of master daemon
- | command = "$JAVA_HOME/bin/java -Xmx512m"
- | memory = "512"
- | vcores = "1"
- | }
- |
- | worker {
- | ## memory of worker daemon
- | command = "$JAVA_HOME/bin/java -Xmx512m"
- | containers = "4"
- | ## This also contains all memory for child executors.
- | memory = "4096"
- | vcores = "1"
- | }
- | services {
- | enabled = true
- | }
- | }
- |}
- """.stripMargin).withFallback(TestUtil.DEFAULT_CONFIG)
-
- "MasterCommand" should "create correct command line" in {
- val version = "gearpump-0.1"
- val master = MasterCommand(config, version, HostPort("127.0.0.1", 8080))
-
- // scalastyle:off line.size.limit
- val expected = "$JAVA_HOME/bin/java -Xmx512m -cp conf:pack/gearpump-0.1/conf:pack/gearpump-0.1/lib/daemon/*:pack/gearpump-0.1/lib/*:$CLASSPATH -Dgearpump.cluster.masters.0=127.0.0.1:8080 -Dgearpump.hostname=127.0.0.1 -Dgearpump.master-resource-manager-container-id={{CONTAINER_ID}} -Dgearpump.home={{LOCAL_DIRS}}/{{CONTAINER_ID}}/pack/gearpump-0.1 -Dgearpump.log.daemon.dir=<LOG_DIR> -Dgearpump.log.application.dir=<LOG_DIR> org.apache.gearpump.cluster.main.Master -ip 127.0.0.1 -port 8080 2>&1 | /usr/bin/tee -a <LOG_DIR>/stderr"
- // scalastyle:on line.size.limit
- assert(master.get == expected)
- }
-
- "WorkerCommand" should "create correct command line" in {
- val version = "gearpump-0.1"
- val worker = WorkerCommand(config, version, HostPort("127.0.0.1", 8080), "worker-machine")
- // scalastyle:off line.size.limit
- val expected = "$JAVA_HOME/bin/java -Xmx512m -cp conf:pack/gearpump-0.1/conf:pack/gearpump-0.1/lib/daemon/*:pack/gearpump-0.1/lib/*:$CLASSPATH -Dgearpump.cluster.masters.0=127.0.0.1:8080 -Dgearpump.log.daemon.dir=<LOG_DIR> -Dgearpump.worker-resource-manager-container-id={{CONTAINER_ID}} -Dgearpump.home={{LOCAL_DIRS}}/{{CONTAINER_ID}}/pack/gearpump-0.1 -Dgearpump.log.application.dir=<LOG_DIR> -Dgearpump.hostname=worker-machine org.apache.gearpump.cluster.main.Worker 2>&1 | /usr/bin/tee -a <LOG_DIR>/stderr"
- // scalastyle:on line.size.limit
- assert(worker.get == expected)
- }
-
- "AppMasterCommand" should "create correct command line" in {
- val version = "gearpump-0.1"
- val appmaster = AppMasterCommand(config, version, Array("arg1", "arg2", "arg3"))
- // scalastyle:off line.size.limit
- val expected = "$JAVA_HOME/bin/java -Xmx512m -cp conf:pack/gearpump-0.1/conf:pack/gearpump-0.1/dashboard:pack/gearpump-0.1/lib/*:pack/gearpump-0.1/lib/daemon/*:pack/gearpump-0.1/lib/services/*:pack/gearpump-0.1/lib/yarn/*:$CLASSPATH -Dgearpump.home={{LOCAL_DIRS}}/{{CONTAINER_ID}}/pack/gearpump-0.1 -Dgearpump.binary-version-with-scala-version=gearpump-0.1 -Dgearpump.log.daemon.dir=<LOG_DIR> -Dgearpump.log.application.dir=<LOG_DIR> -Dgearpump.hostname={{NM_HOST}} org.apache.gearpump.experiments.yarn.appmaster.YarnAppMaster arg1 arg2 arg3 2>&1 | /usr/bin/tee -a <LOG_DIR>/stderr"
- // scalastyle:on line.size.limit
- assert(appmaster.get == expected)
- }
-}
\ No newline at end of file
diff --git a/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/client/LaunchClusterSpec.scala b/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/client/LaunchClusterSpec.scala
index c0a4ee2..3f2f9cb 100644
--- a/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/client/LaunchClusterSpec.scala
+++ b/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/client/LaunchClusterSpec.scala
@@ -186,7 +186,7 @@
// val workerResources = ArgumentCaptor.forClass(classOf[List[Resource]])
// scalastyle:off line.size.limit
- val expectedCommand = "$JAVA_HOME/bin/java -Xmx512m -cp conf:pack/gearpump-0.2/conf:pack/gearpump-0.2/dashboard:pack/gearpump-0.2/lib/*:pack/gearpump-0.2/lib/daemon/*:pack/gearpump-0.2/lib/services/*:pack/gearpump-0.2/lib/yarn/*:$CLASSPATH -Dgearpump.home={{LOCAL_DIRS}}/{{CONTAINER_ID}}/pack/gearpump-0.2 -Dgearpump.binary-version-with-scala-version=gearpump-0.2 -Dgearpump.log.daemon.dir=<LOG_DIR> -Dgearpump.log.application.dir=<LOG_DIR> -Dgearpump.hostname={{NM_HOST}} org.apache.gearpump.experiments.yarn.appmaster.YarnAppMaster -conf /root/.gearpump_application_0_0000/conf/ -package gearpump.zip 2>&1 | /usr/bin/tee -a <LOG_DIR>/stderr"
+ val expectedCommand = "$JAVA_HOME/bin/java -Xmx512m -noverify -cp conf:pack/gearpump-0.2/conf:pack/gearpump-0.2/dashboard:pack/gearpump-0.2/lib/*:pack/gearpump-0.2/lib/daemon/*:pack/gearpump-0.2/lib/services/*:pack/gearpump-0.2/lib/yarn/*:$CLASSPATH -Dgearpump.home={{LOCAL_DIRS}}/{{CONTAINER_ID}}/pack/gearpump-0.2 -Dgearpump.binary-version-with-scala-version=gearpump-0.2 -Dgearpump.log.daemon.dir=<LOG_DIR> -Dgearpump.log.application.dir=<LOG_DIR> -Dgearpump.hostname={{NM_HOST}} org.apache.gearpump.experiments.yarn.appmaster.YarnAppMaster -conf /root/.gearpump_application_0_0000/conf/ -package gearpump.zip 2>&1 | /usr/bin/tee -a <LOG_DIR>/stderr"
// scalastyle:on line.size.limit
verify(yarnClient).submit("gearpump", appId, expectedCommand,
Resource.newInstance(512, 1), "default",
diff --git a/project/BuildExperiments.scala b/project/BuildExperiments.scala
index 3bebf22..550b407 100644
--- a/project/BuildExperiments.scala
+++ b/project/BuildExperiments.scala
@@ -115,6 +115,6 @@
id = "gearpump-experimental-cgroup",
base = file("experiments/cgroup"),
settings = commonSettings ++ noPublish)
- .dependsOn (core % "test->test; compile->compile")
+ .dependsOn (core % "provided")
.disablePlugins(sbtassembly.AssemblyPlugin)
}
\ No newline at end of file
diff --git a/project/BuildGearpump.scala b/project/BuildGearpump.scala
index 793d903..f0c9517 100644
--- a/project/BuildGearpump.scala
+++ b/project/BuildGearpump.scala
@@ -221,7 +221,7 @@
"org.apache.hadoop" % "hadoop-common" % hadoopVersion
)
)
- ).dependsOn(core % "compile->compile").disablePlugins(sbtassembly.AssemblyPlugin)
+ ).dependsOn(core % "provided").disablePlugins(sbtassembly.AssemblyPlugin)
private def changeShadedDeps(toExclude: Set[String], toInclude: List[xml.Node],
node: xml.Node): xml.Node = {
