upgrade mongo-elasticsearch-sync to es 2.0 + docker
needs updates to streams-persist-mongo from STREAMS-426 to merge first
diff --git a/local/mongo-elasticsearch-sync/pom.xml b/local/mongo-elasticsearch-sync/pom.xml
index d268ed7..2dde464 100644
--- a/local/mongo-elasticsearch-sync/pom.xml
+++ b/local/mongo-elasticsearch-sync/pom.xml
@@ -33,8 +33,8 @@
<properties>
<docker.repo>apachestreams</docker.repo>
- <elasticsearch.version>1.1.0</elasticsearch.version>
- <lucene.version>4.7.2</lucene.version>
+ <elasticsearch.version>2.3.5</elasticsearch.version>
+ <lucene.version>5.5.0</lucene.version>
</properties>
<dependencies>
@@ -94,6 +94,13 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
+ <artifactId>streams-persist-mongo</artifactId>
+ <version>0.4-incubating-SNAPSHOT</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
<artifactId>streams-pojo</artifactId>
<version>0.4-incubating-SNAPSHOT</version>
<type>test-jar</type>
@@ -180,32 +187,6 @@
</configuration>
</plugin>
<plugin>
- <groupId>com.github.joelittlejohn.embedmongo</groupId>
- <artifactId>embedmongo-maven-plugin</artifactId>
- <version>0.1.12</version>
- <executions>
- <execution>
- <id>start</id>
- <goals>
- <goal>start</goal>
- </goals>
- <configuration>
- <port>37017</port>
- <version>2.4.0</version>
- <databaseDirectory>target/mongotest</databaseDirectory>
- <logging>console</logging>
- <skip>false</skip>
- </configuration>
- </execution>
- <execution>
- <id>stop</id>
- <goals>
- <goal>stop</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
</plugin>
@@ -252,35 +233,43 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
- <version>2.4</version>
+ <configuration>
+ <includes>**/*.json,**/*.conf</includes>
+ <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+ <includeGroupIds>org.apache.streams</includeGroupIds>
+ <includeTypes>test-jar</includeTypes>
+ </configuration>
<executions>
<execution>
- <id>resource-dependencies</id>
+ <id>test-resource-dependencies</id>
<phase>process-test-resources</phase>
<goals>
<goal>unpack-dependencies</goal>
</goals>
- <configuration>
- <includeArtifactIds>streams-pojo</includeArtifactIds>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- </configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
- <version>2.12.4</version>
- <executions>
- <execution>
- <id>integration-tests</id>
- <goals>
- <goal>integration-test</goal>
- <goal>verify</goal>
- </goals>
- </execution>
- </executions>
+ <configuration>
+ <!-- Run integration test suite rather than individual tests. -->
+ <excludes>
+ <exclude>**/*Test.java</exclude>
+ <exclude>**/*Tests.java</exclude>
+ <exclude>**/*IT.java</exclude>
+ </excludes>
+ <includes>
+ <include>**/*ITs.java</include>
+ </includes>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.surefire</groupId>
+ <artifactId>surefire-junit47</artifactId>
+ <version>${failsafe.plugin.version}</version>
+ </dependency>
+ </dependencies>
</plugin>
<plugin>
<groupId>io.fabric8</groupId>
@@ -289,4 +278,93 @@
</plugins>
</build>
+ <profiles>
+ <profile>
+ <id>dockerITs</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ <property>
+ <name>skipITs</name>
+ <value>false</value>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>io.fabric8</groupId>
+ <artifactId>docker-maven-plugin</artifactId>
+ <version>${docker.plugin.version}</version>
+ <configuration combine.self="override">
+ <watchInterval>500</watchInterval>
+ <logDate>default</logDate>
+ <verbose>true</verbose>
+ <autoPull>on</autoPull>
+ <images>
+ <image>
+ <name>elasticsearch:2.3.5</name>
+ <alias>elasticsearch</alias>
+ <run>
+ <namingStrategy>none</namingStrategy>
+ <ports>
+ <port>${es.http.host}:${es.http.port}:9200</port>
+ <port>${es.tcp.host}:${es.tcp.port}:9300</port>
+ </ports>
+ <portPropertyFile>elasticsearch.properties</portPropertyFile>
+ <wait>
+ <log>elasticsearch startup</log>
+ <http>
+ <url>http://${es.http.host}:${es.http.port}</url>
+ <method>GET</method>
+ <status>200</status>
+ </http>
+ <time>20000</time>
+ <kill>1000</kill>
+ <shutdown>500</shutdown>
+ <!--<tcp>-->
+ <!--<host>${es.transport.host}</host>-->
+ <!--<ports>-->
+ <!--<port>${es.transport.port}</port>-->
+ <!--</ports>-->
+ <!--</tcp>-->
+ </wait>
+ <log>
+ <enabled>true</enabled>
+ <date>default</date>
+ <color>cyan</color>
+ </log>
+ </run>
+ <watch>
+ <mode>none</mode>
+ </watch>
+ </image>
+ <image>
+ <name>mongo:3.2.0</name>
+ <alias>mongo</alias>
+ <run>
+ <namingStrategy>none</namingStrategy>
+ <ports>
+ <port>${mongo.tcp.host}:${mongo.tcp.port}:27017</port>
+ </ports>
+ <portPropertyFile>mongo.properties</portPropertyFile>
+ <log>
+ <enabled>true</enabled>
+ <date>default</date>
+ <color>cyan</color>
+ </log>
+ </run>
+ <watch>
+ <mode>none</mode>
+ </watch>
+ </image>
+ </images>
+ </configuration>
+
+ </plugin>
+
+ </plugins>
+ </build>
+
+ </profile>
+ </profiles>
+
</project>
\ No newline at end of file
diff --git a/local/mongo-elasticsearch-sync/src/main/resources/application.json b/local/mongo-elasticsearch-sync/src/main/resources/application.json
deleted file mode 100644
index 99cc598..0000000
--- a/local/mongo-elasticsearch-sync/src/main/resources/application.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "source": {
- "host": "localhost",
- "port": 27017,
- "db": "streams",
- "collection": "activities"
- },
- "destination": {
- "hosts": [
- "localhost"
- ],
- "port": 9300,
- "clusterName": "elasticsearch",
- "index": "destination",
- "type": "activity"
- }
-}
diff --git a/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/elasticsearch/test/MongoElasticsearchSyncIT.java b/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/elasticsearch/test/MongoElasticsearchSyncIT.java
deleted file mode 100644
index 50f9c4c..0000000
--- a/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/elasticsearch/test/MongoElasticsearchSyncIT.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.example.elasticsearch.test;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.Charsets;
-import org.apache.commons.io.IOUtils;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.elasticsearch.example.MongoElasticsearchSync;
-import org.apache.streams.example.elasticsearch.MongoElasticsearchSyncConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.mongo.MongoPersistWriter;
-import org.apache.streams.pojo.json.Activity;
-import org.elasticsearch.test.ElasticsearchIntegrationTest;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.util.List;
-
-/**
- * Test copying documents between two indexes on same cluster
- */
-@ElasticsearchIntegrationTest.ClusterScope(scope= ElasticsearchIntegrationTest.Scope.TEST, numNodes=1)
-public class MongoElasticsearchSyncIT extends ElasticsearchIntegrationTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(MongoElasticsearchSyncIT.class);
-
- ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- MongoElasticsearchSyncConfiguration syncConfiguration;
-
- int srcCount = 0;
-
- @Before
- public void prepareTest() throws Exception {
-
- syncConfiguration = MAPPER.readValue(
- MongoElasticsearchSyncIT.class.getResourceAsStream("/testSync.json"), MongoElasticsearchSyncConfiguration.class);
-
- syncConfiguration.getDestination().setClusterName(cluster().getClusterName());
-
- MongoPersistWriter setupWriter = new MongoPersistWriter(syncConfiguration.getSource());
-
- setupWriter.prepare(null);
-
- InputStream testActivityFolderStream = MongoElasticsearchSyncIT.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, Charsets.UTF_8);
-
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = MongoElasticsearchSyncIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- activity.getAdditionalProperties().remove("$license");
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- setupWriter.write( datum );
- LOGGER.info("Wrote: " + activity.getVerb() );
- srcCount++;
- }
-
- setupWriter.cleanUp();
-
- }
-
- @Test
- public void testSync() throws Exception {
-
- assert srcCount > 0;
-
- MongoElasticsearchSync sync = new MongoElasticsearchSync(syncConfiguration);
-
- Thread reindexThread = new Thread(sync);
- reindexThread.start();
- reindexThread.join();
-
- flushAndRefresh();
-
- assert(indexExists("destination"));
-
- long destCount = client().count(client().prepareCount("destination").request()).get().getCount();
- assert srcCount == destCount;
-
- }
-}
diff --git a/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/mongodb/test/MongoElasticsearchSyncIT.java b/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/mongodb/test/MongoElasticsearchSyncIT.java
new file mode 100644
index 0000000..5ebc204
--- /dev/null
+++ b/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/mongodb/test/MongoElasticsearchSyncIT.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.example.mongodb.test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigParseOptions;
+import org.apache.commons.io.Charsets;
+import org.apache.commons.io.IOUtils;
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.elasticsearch.ElasticsearchClientManager;
+import org.apache.streams.elasticsearch.example.MongoElasticsearchSync;
+import org.apache.streams.example.elasticsearch.MongoElasticsearchSyncConfiguration;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Properties;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+
+/**
+ * Test copying documents between two indexes on same cluster
+ */
+public class MongoElasticsearchSyncIT {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(MongoElasticsearchSyncIT.class);
+
+ ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+
+ protected MongoElasticsearchSyncConfiguration testConfiguration;
+ protected Client testClient;
+
+ @Before
+ public void prepareTest() throws Exception {
+
+ Config reference = ConfigFactory.load();
+ File conf_file = new File("target/test-classes/MongoElasticsearchSyncIT.conf");
+ assert(conf_file.exists());
+ Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
+ Properties mongo_properties = new Properties();
+ InputStream mongo_stream = new FileInputStream("mongo.properties");
+ mongo_properties.load(mongo_stream);
+ Config mongoProps = ConfigFactory.parseProperties(mongo_properties);
+ Properties es_properties = new Properties();
+ InputStream es_stream = new FileInputStream("elasticsearch.properties");
+ es_properties.load(es_stream);
+ Config esProps = ConfigFactory.parseProperties(es_properties);
+ Config typesafe = testResourceConfig.withFallback(mongoProps).withFallback(esProps).withFallback(reference).resolve();
+ StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe);
+ testConfiguration = new ComponentConfigurator<>(MongoElasticsearchSyncConfiguration.class).detectConfiguration(typesafe);
+ testClient = new ElasticsearchClientManager(testConfiguration.getDestination()).getClient();
+
+ ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest();
+ ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet();
+ assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED);
+
+ IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getDestination().getIndex());
+ IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet();
+ assertFalse(indicesExistsResponse.isExists());
+ }
+
+ @Test
+ public void testSync() throws Exception {
+
+ MongoElasticsearchSync sync = new MongoElasticsearchSync(testConfiguration);
+
+ sync.run();
+
+ IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getDestination().getIndex());
+ IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet();
+ assertTrue(indicesExistsResponse.isExists());
+
+ // assert lines in file
+ SearchRequestBuilder countRequest = testClient
+ .prepareSearch(testConfiguration.getDestination().getIndex())
+ .setTypes(testConfiguration.getDestination().getType());
+ SearchResponse countResponse = countRequest.execute().actionGet();
+
+ assertEquals(89, (int)countResponse.getHits().getTotalHits());
+
+ }
+}
diff --git a/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/mongodb/test/SyncITs.java b/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/mongodb/test/SyncITs.java
new file mode 100644
index 0000000..7ba67a5
--- /dev/null
+++ b/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/mongodb/test/SyncITs.java
@@ -0,0 +1,16 @@
+package org.apache.streams.example.mongodb.test;
+
+import org.apache.streams.mongo.test.MongoPersistIT;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ MongoPersistIT.class,
+ MongoElasticsearchSyncIT.class
+})
+
+public class SyncITs {
+ // the class remains empty,
+ // used only as a holder for the above annotations
+}
\ No newline at end of file
diff --git a/local/mongo-elasticsearch-sync/src/test/resources/MongoElasticsearchSyncIT.conf b/local/mongo-elasticsearch-sync/src/test/resources/MongoElasticsearchSyncIT.conf
new file mode 100644
index 0000000..c178e8f
--- /dev/null
+++ b/local/mongo-elasticsearch-sync/src/test/resources/MongoElasticsearchSyncIT.conf
@@ -0,0 +1,21 @@
+{
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "source": {
+ host = ${mongo.tcp.host}
+ port = ${mongo.tcp.port}
+ "db": "mongo_persist_it",
+ "collection": "activity"
+ },
+ "destination": {
+ hosts = [
+ ${es.tcp.host}
+ ]
+ port = ${es.tcp.port}
+ "clusterName": "elasticsearch",
+ "index": "mongo_elasticsearch_sync_it",
+ "type": "activity",
+ "forceUseConfig": true
+ }
+}
