[GEARPUMP-361] Build gearpump docker image locally
Author: manuzhang <owenzhang1990@gmail.com>
Closes #233 from manuzhang/fix-it.
diff --git a/integrationtest/README.md b/integrationtest/README.md
index b626c6a..28ca6b9 100644
--- a/integrationtest/README.md
+++ b/integrationtest/README.md
@@ -3,8 +3,6 @@
To run the integration test, you need a Linux with Kernel version >= 3.10 and Docker 1.7 (or higher). The test framework will use several Docker images.
These docker image **NEED** to be prepared **BEFOREHAND** to avoid timeout during testing:
- * [The Gearpump Cluster Launcher and Storm Client](https://hub.docker.com/r/grubykarol/gearpump-launcher/)
- `docker pull grubykarol/gearpump-launcher`
* [The standalone single node Kafka cluster with Zookeeper](https://hub.docker.com/r/grubykarol/kafka/)
`docker pull grubykarol/kafka:0.8.2.1`
* [The Hadoop image](https://hub.docker.com/r/sequenceiq/hadoop-docker/)
@@ -29,7 +27,9 @@
`git clone https://github.com/apache/incubator-gearpump.git`
2. Build Gearpump project
`sbt assembly pack`
-3. Run Integration test
+3. Build Gearpump docker image
+ `docker build -t gearpump/gearpump-launcher integrationtest/docker/gearpump`
+4. Run Integration test
`sbt it:test`
The test will launch a Gearpump cluster with 1 master and 2 worker nodes as 3 Docker containers. It might take 10-20 minutes to go through all the test cases. It depends on, how powerful your machine is. The Docker container itself does not have a Gearpump distribution. It will link your local build to the Docker container. When tests are finished, you can see the test result on the screen, or you can save them to a file with this command `sbt it:test > test_report.out`. To investigate Gearpump log, please check the directory `output/target/pack/logs`.
@@ -69,15 +69,15 @@
export GEARPUMP_HOME=/path/to/gearpump/dist
## Start Master node
-docker run -d -h master0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --name master0 grubykarol/gearpump-launcher master -ip master0 -port 3000
+docker run -d -h master0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --name master0 gearpump/gearpump-launcher master -ip master0 -port 3000
## Start Worker0 node
-docker run -d -h worker0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 grubykarol/gearpump-launcher worker
+docker run -d -h worker0 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 gearpump/gearpump-launcher worker
## ...
## Start Worker1 node
-docker run -d -h worker1 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 grubykarol/gearpump-launcher worker
+docker run -d -h worker1 -v /etc/localtime:/etc/localtime:ro -e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 -v $GEARPUMP_HOME:/opt/gearpump -v /tmp/gearpump:/var/log/gearpump --link master0 --name worker0 gearpump/gearpump-launcher worker
```
diff --git a/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala b/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala
index 6e32f63..f1819df 100644
--- a/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala
+++ b/integrationtest/core/src/main/scala/org/apache/gearpump/integrationtest/minicluster/BaseContainer.scala
@@ -28,15 +28,9 @@
masterAddrs: List[(String, Int)],
tunnelPorts: Set[Int] = Set.empty) {
- private val IMAGE_NAME = "grubykarol/gearpump-launcher"
+ private val IMAGE_NAME = "gearpump/gearpump-launcher"
private val DOCKER_IMAGE_GEARPUMP_HOME = "/opt/gearpump"
- private val DOCKER_IMAGE_LOG_HOME = "/var/log/gearpump"
private val HOST_GEARPUMP_HOME = "pwd".!!.trim + "/output/target/pack"
- private val HOST_LOG_HOME = {
- val dir = "/tmp/gearpump"
- s"mkdir -p $dir".!!
- s"mktemp -p $dir -d".!!.trim
- }
private val CLUSTER_OPTS = {
masterAddrs.zipWithIndex.map { case (hostPort, index) =>
@@ -48,8 +42,7 @@
Docker.createAndStartContainer(host, IMAGE_NAME, command,
environ = Map("JAVA_OPTS" -> CLUSTER_OPTS),
volumes = Map(
- HOST_GEARPUMP_HOME -> DOCKER_IMAGE_GEARPUMP_HOME,
- HOST_LOG_HOME -> DOCKER_IMAGE_LOG_HOME),
+ HOST_GEARPUMP_HOME -> DOCKER_IMAGE_GEARPUMP_HOME),
knownHosts = masterAddrs.map(_._1).filter(_ != host).toSet,
tunnelPorts = tunnelPorts)
}
diff --git a/integrationtest/docker/gearpump/Dockerfile b/integrationtest/docker/gearpump/Dockerfile
index 754f6de..0c5042e 100644
--- a/integrationtest/docker/gearpump/Dockerfile
+++ b/integrationtest/docker/gearpump/Dockerfile
@@ -18,7 +18,7 @@
FROM openjdk:8u131-jre-alpine
# Install python
-RUN apk add --no-cache python && \
+RUN apk add --update curl python && \
python -m ensurepip && \
rm -r /usr/lib/python*/ensurepip && \
pip install --upgrade pip setuptools && \
diff --git a/integrationtest/docker/gearpump/README.md b/integrationtest/docker/gearpump/README.md
index bc38ccb..26820b5 100644
--- a/integrationtest/docker/gearpump/README.md
+++ b/integrationtest/docker/gearpump/README.md
@@ -13,14 +13,14 @@
-v $GEARPUMP_HOME:/opt/gearpump \
-e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 \
-p 8090:8090 \
- grubykarol/gearpump-launcher \
+ gearpump/gearpump-launcher \
master -ip master0 -port 3000
docker run -d \
--link master0 \
-v $GEARPUMP_HOME:/opt/gearpump \
-e JAVA_OPTS=-Dgearpump.cluster.masters.0=master0:3000 \
- grubykarol/gearpump-launcher \
+ gearpump/gearpump-launcher \
worker
docker exec master0 bin/gear info
diff --git a/integrationtest/docker/gearpump/start.sh b/integrationtest/docker/gearpump/start.sh
index 76d1f18..987adfd 100644
--- a/integrationtest/docker/gearpump/start.sh
+++ b/integrationtest/docker/gearpump/start.sh
@@ -26,14 +26,6 @@
exit 1
fi
-update_config_file() {
- CONF_FILE="$SUT_HOME"/conf/gear.conf
- mkdir /var/log/gearpump
- sed -i 's/log\.daemon\.dir\s*=.*$/log.daemon.dir = "\/var\/log\/gearpump\/daemon"/g' $CONF_FILE
- sed -i 's/log\.application\.dir\s*=.*$/log.application.dir = "\/var\/log\/gearpump\/app"/g' $CONF_FILE
- sed -i 's/#\s*jarstore\.rootpath\s*=.*$/jarstore.rootpath = "\/tmp"/g' $CONF_FILE
-}
-
set_and_export_java_opts() {
JAVA_OPTS="$JAVA_OPTS $*"
export JAVA_OPTS
@@ -46,7 +38,6 @@
master|local)
# Launch a container with Gearpump cluster and REST interface (in foreground)
HOSTNAME=$(hostname)
- update_config_file
set_and_export_java_opts \
"-Dgearpump.hostname=$HOSTNAME" \
"-Dgearpump.services.host=$HOSTNAME"
@@ -55,7 +46,6 @@
;;
worker)
# Launch a container with a Gearpump worker (in foreground)
- update_config_file
set_and_export_java_opts \
"-Dgearpump.hostname=$(hostname -i)"
nohup sh "$SUT_HOME"/bin/worker
@@ -63,7 +53,6 @@
gear|storm)
# Launch a container and execute command `gear` or `storm`
# Container will be killed, when command is executed.
- update_config_file
set_and_export_java_opts \
"-Dgearpump.hostname=$(hostname -i)"
sh "$SUT_HOME"/bin/"$COMMAND" "$@"
