GORA-669 Improve tests stability on Jenkins (#233)
* Synchronize Maven options in Github Actions (`.github/workflows/master-pr-build.yml`) and Jenkins build (`Jenkinsfile`)
* Upgrade testcontainers to 1.14.3
* Use upstream `MongoDBContainer` and remove our old `MongoContainer` implementation
* Rework our `*LogWaitStrategy` to reduce boilerplate code
* Improve stability of our `GoraMongodbAuthenticationTestDriver` by assuring that "waiting for connections" log message if present twice (because MongoDB docker container restart daemon itself)
* Reduce log verbosity of gora-solr tests
* Enforce a global `forkedProcessTimeoutInSeconds` of 40 minutes in pom.xml
diff --git a/.github/workflows/master-pr-build.yml b/.github/workflows/master-pr-build.yml
index 5d2a207..5ab3b17 100644
--- a/.github/workflows/master-pr-build.yml
+++ b/.github/workflows/master-pr-build.yml
@@ -21,6 +21,10 @@
pull_request:
branches:
- master
+
+env:
+ MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version --no-transfer-progress"
+
jobs:
build:
runs-on: ubuntu-latest
@@ -28,10 +32,14 @@
matrix:
java: [ '1.8' ]
steps:
- - uses: actions/checkout@v1
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v1
- with:
- java-version: ${{ matrix.java }}
- - name: mvn sourcecheck
- run: mvn --no-transfer-progress --batch-mode clean install
\ No newline at end of file
+ - uses: actions/checkout@v2
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v1
+ with:
+ java-version: ${{ matrix.java }}
+
+ - name: Build
+ run: mvn $MAVEN_CLI_OPTS -DskipTests clean install
+
+ - name: Test
+ run: mvn $MAVEN_CLI_OPTS verify
\ No newline at end of file
diff --git a/.github/workflows/master-push-build.yml b/.github/workflows/master-push-build.yml
index 3e3c627..517f455 100644
--- a/.github/workflows/master-push-build.yml
+++ b/.github/workflows/master-push-build.yml
@@ -21,6 +21,10 @@
push:
branches:
- master
+
+env:
+ MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version --no-transfer-progress"
+
jobs:
build:
runs-on: ubuntu-latest
@@ -28,10 +32,14 @@
matrix:
java: [ '1.8' ]
steps:
- - uses: actions/checkout@v1
+ - uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- - name: mvn sourcecheck
- run: mvn --no-transfer-progress --batch-mode clean install
\ No newline at end of file
+
+ - name: Build
+ run: mvn $MAVEN_CLI_OPTS -DskipTests clean install
+
+ - name: Test
+ run: mvn $MAVEN_CLI_OPTS verify
\ No newline at end of file
diff --git a/Jenkinsfile b/Jenkinsfile
index b3eebee..cbae42f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,7 +50,7 @@
// only keep the latest 10 builds
buildDiscarder logRotator(numToKeepStr: '10')
// cancel build if not complete within two hours of scheduling
- timeout time: 2, unit: 'HOURS'
+ timeout time: 3, unit: 'HOURS'
disableConcurrentBuilds()
}
@@ -86,7 +86,7 @@
post {
always {
- junit '**/target/surefire-reports/TEST-*.xml'
+ junit testResults: '**/target/surefire-reports/TEST-*.xml', testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
}
}
}
diff --git a/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java
index b8f281e..58c1654 100644
--- a/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java
+++ b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java
@@ -22,11 +22,10 @@
import org.apache.gora.mapreduce.MapReduceTestUtils;
import org.apache.gora.store.DataStoreFactory;
import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-import org.junit.Before;
import org.junit.After;
+import org.junit.Before;
import org.junit.ClassRule;
-
+import org.junit.Test;
import org.testcontainers.containers.GenericContainer;
import java.time.Duration;
diff --git a/gora-aerospike/src/test/java/org/apache/gora/aerospike/store/AerospikeStartupLogWaitStrategy.java b/gora-aerospike/src/test/java/org/apache/gora/aerospike/store/AerospikeStartupLogWaitStrategy.java
index 8b9cf5e..4881f93 100644
--- a/gora-aerospike/src/test/java/org/apache/gora/aerospike/store/AerospikeStartupLogWaitStrategy.java
+++ b/gora-aerospike/src/test/java/org/apache/gora/aerospike/store/AerospikeStartupLogWaitStrategy.java
@@ -18,36 +18,17 @@
package org.apache.gora.aerospike.store;
-import org.testcontainers.containers.ContainerLaunchException;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.output.OutputFrame;
-import org.testcontainers.containers.output.WaitingConsumer;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.function.Predicate;
-
-public class AerospikeStartupLogWaitStrategy extends GenericContainer.AbstractWaitStrategy {
+public class AerospikeStartupLogWaitStrategy extends LogMessageWaitStrategy {
private static final String regEx = ".*heartbeat-received: self 0 foreign 0.*";
private int times = 2;
- protected void waitUntilReady() {
- WaitingConsumer waitingConsumer = new WaitingConsumer();
- this.container.followOutput(waitingConsumer);
- Predicate waitPredicate = (outputFrame) -> {
- String trimmedFrameText = ((OutputFrame) outputFrame).getUtf8String().replaceFirst("\n$", "");
- return trimmedFrameText.matches(regEx);
- };
-
- try {
- waitingConsumer.waitUntil(waitPredicate, this.startupTimeout.getSeconds(), TimeUnit.SECONDS,
- this.times);
- } catch (TimeoutException var4) {
- throw new ContainerLaunchException(
- "Timed out waiting for log output matching Aerospike server startup Log \'" + regEx
- + "\'");
- }
+ public AerospikeStartupLogWaitStrategy() {
+ withRegEx(regEx);
+ withTimes(times);
}
+
}
\ No newline at end of file
diff --git a/gora-benchmark/pom.xml b/gora-benchmark/pom.xml
index c8d6e9d..811cd68 100644
--- a/gora-benchmark/pom.xml
+++ b/gora-benchmark/pom.xml
@@ -188,9 +188,8 @@
</dependency>
<dependency>
- <groupId>org.apache.gora</groupId>
- <artifactId>gora-mongodb</artifactId>
- <type>test-jar</type>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mongodb</artifactId>
<scope>test</scope>
</dependency>
@@ -199,5 +198,6 @@
<groupId>org.ektorp</groupId>
<artifactId>org.ektorp</artifactId>
</dependency>
+
</dependencies>
</project>
diff --git a/gora-benchmark/src/test/java/org/apache/gora/benchmark/GoraClientTest.java b/gora-benchmark/src/test/java/org/apache/gora/benchmark/GoraClientTest.java
index 91a7c07..7ce0127 100644
--- a/gora-benchmark/src/test/java/org/apache/gora/benchmark/GoraClientTest.java
+++ b/gora-benchmark/src/test/java/org/apache/gora/benchmark/GoraClientTest.java
@@ -17,9 +17,7 @@
*/
package org.apache.gora.benchmark;
-import com.mongodb.ServerAddress;
import org.apache.gora.benchmark.generated.User;
-import org.apache.gora.mongodb.MongoContainer;
import org.apache.gora.store.DataStoreFactory;
import org.apache.gora.util.GoraException;
import org.junit.After;
@@ -27,6 +25,7 @@
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.MongoDBContainer;
import site.ycsb.ByteIterator;
import site.ycsb.Status;
import site.ycsb.StringByteIterator;
@@ -50,7 +49,7 @@
private static HashMap<String, ByteIterator> DATA_TO_UPDATE;
private static HashMap<String, ByteIterator> INTEGER_DATA;
private static boolean isMongoDBSetupDone = false;
- private static MongoContainer mongo;
+ private static MongoDBContainer mongo;
/**
* Setup MongoDB embed cluster. This function will auto provision a MongoDB
@@ -60,11 +59,10 @@
private void setupMongoDBCluster() {
try {
if (!isMongoDBSetupDone) {
- mongo = new MongoContainer("3.6");
+ mongo = new MongoDBContainer("mongo:3.6");
mongo.start();
}
- ServerAddress address = mongo.getServerAddress();
- LOG.info("Started MongoDB Server on " + address.getHost() + ":" + address.getPort());
+ LOG.info("Started MongoDB Server on " + mongo.getReplicaSetUrl());
isMongoDBSetupDone = true;
} catch (Exception e) {
LOG.info("Cannot Start MongoDB Server {}", e.getMessage(), e);
@@ -99,8 +97,7 @@
String dataStoreToTest = GoraBenchmarkUtils.getDataBase(dataStoreProperties);
if (Constants.MONGODB.equals(dataStoreToTest)) {
setupMongoDBCluster();
- ServerAddress address = mongo.getServerAddress();
- properties.setProperty("gora.mongodb.servers", address.getHost() + ":" + address.getPort());
+ properties.setProperty("gora.mongodb.servers", mongo.getContainerIpAddress() + ":" + mongo.getFirstMappedPort());
}
benchmarkClient = new GoraBenchmarkClient();
diff --git a/gora-couchdb/pom.xml b/gora-couchdb/pom.xml
index 4ff0b93..e1add43 100644
--- a/gora-couchdb/pom.xml
+++ b/gora-couchdb/pom.xml
@@ -154,12 +154,6 @@
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>runtime</scope>
diff --git a/gora-couchdb/src/test/java/org/apache/gora/couchdb/CouchDBStartupLogWaitStrategy.java b/gora-couchdb/src/test/java/org/apache/gora/couchdb/CouchDBStartupLogWaitStrategy.java
index 7cd06cf..6fc043d 100644
--- a/gora-couchdb/src/test/java/org/apache/gora/couchdb/CouchDBStartupLogWaitStrategy.java
+++ b/gora-couchdb/src/test/java/org/apache/gora/couchdb/CouchDBStartupLogWaitStrategy.java
@@ -18,43 +18,19 @@
package org.apache.gora.couchdb;
-import org.testcontainers.containers.ContainerLaunchException;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.output.OutputFrame;
-import org.testcontainers.containers.output.WaitingConsumer;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.function.Predicate;
/**
* Log based CouchDB server startup wait strategy to sync server
* startup to test suit startup.
*/
-public class CouchDBStartupLogWaitStrategy extends GenericContainer.AbstractWaitStrategy {
+public class CouchDBStartupLogWaitStrategy extends LogMessageWaitStrategy {
private static final String regEx = ".*Apache CouchDB has started. Time to relax..*";
- private int times = 1;
-
- protected void waitUntilReady() {
- WaitingConsumer waitingConsumer = new WaitingConsumer();
- this.container.followOutput(waitingConsumer);
- Predicate waitPredicate = (outputFrame) -> {
- String trimmedFrameText = ((OutputFrame) outputFrame)
- .getUtf8String()
- .replaceFirst("\n$", "");
- return trimmedFrameText.matches(regEx);
- };
-
- try {
- waitingConsumer.waitUntil(waitPredicate, this.startupTimeout.getSeconds(), TimeUnit.SECONDS,
- this.times);
- } catch (TimeoutException var4) {
- throw new ContainerLaunchException(
- "Timed out waiting for log output matching CouchDB server startup Log \'" + regEx
- + "\'");
- }
+ public CouchDBStartupLogWaitStrategy() {
+ withRegEx(regEx);
}
}
\ No newline at end of file
diff --git a/gora-hive/src/test/java/org/apache/gora/hive/store/TestHiveStore.java b/gora-hive/src/test/java/org/apache/gora/hive/store/TestHiveStore.java
index 0d635f2..003d8ec 100644
--- a/gora-hive/src/test/java/org/apache/gora/hive/store/TestHiveStore.java
+++ b/gora-hive/src/test/java/org/apache/gora/hive/store/TestHiveStore.java
@@ -35,6 +35,7 @@
import org.apache.gora.util.GoraException;
import org.apache.gora.util.StringUtils;
import org.junit.Ignore;
+import org.junit.Test;
/**
* HiveStore Tests extending {@link DataStoreTestBase} which run the base JUnit test suite for
diff --git a/gora-mongodb/pom.xml b/gora-mongodb/pom.xml
index e802e0f..bd43ae5 100644
--- a/gora-mongodb/pom.xml
+++ b/gora-mongodb/pom.xml
@@ -193,6 +193,13 @@
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>mongodb</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<!-- END of Testing Dependencies -->
</dependencies>
diff --git a/gora-mongodb/src/test/conf/log4j.properties b/gora-mongodb/src/test/conf/log4j.properties
new file mode 100644
index 0000000..0256aa9
--- /dev/null
+++ b/gora-mongodb/src/test/conf/log4j.properties
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, Console
+
+log4j.logger.org.testcontainers=INFO
+log4j.logger.com.github.dockerjava=WARN
+
+log4j.appender.Console=org.apache.log4j.ConsoleAppender
+log4j.appender.Console.layout=org.apache.log4j.PatternLayout
+
+log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %x %-5p [%c{3}] [%t] %m%n
+
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/GoraMongodbTestDriver.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/GoraMongodbTestDriver.java
index 73a60bb..f54b0bb 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/GoraMongodbTestDriver.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/GoraMongodbTestDriver.java
@@ -18,12 +18,12 @@
package org.apache.gora.mongodb;
import com.mongodb.MongoClient;
-import com.mongodb.ServerAddress;
import org.apache.gora.GoraTestDriver;
import org.apache.gora.mongodb.store.MongoStore;
import org.apache.gora.mongodb.store.MongoStoreParameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.MongoDBContainer;
/**
* Driver to set up an embedded MongoDB database instance for use in our
@@ -34,13 +34,13 @@
private static Logger log = LoggerFactory
.getLogger(GoraMongodbTestDriver.class);
- private MongoContainer _container;
+ private MongoDBContainer _container;
private MongoClient _mongo;
/**
* Constructor for this class.
*/
- public GoraMongodbTestDriver(MongoContainer startedContainer) {
+ public GoraMongodbTestDriver(MongoDBContainer startedContainer) {
super(MongoStore.class);
this._container = startedContainer;
}
@@ -50,9 +50,8 @@
*/
@Override
public void setUpClass() {
- ServerAddress address = _container.getServerAddress();
- int port = address.getPort();
- String host = address.getHost();
+ int port = _container.getMappedPort(27017);
+ String host = _container.getContainerIpAddress();
// Store Mongo server "host:port" in Hadoop configuration
// so that MongoStore will be able to get it latter
@@ -64,8 +63,6 @@
*/
@Override
public void tearDownClass() {
- log.info("Shutting down mongodb server...");
- _container.stop();
}
}
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/MongoContainer.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/MongoContainer.java
deleted file mode 100644
index 231d82c..0000000
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/MongoContainer.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.mongodb;
-
-import com.mongodb.ServerAddress;
-import org.testcontainers.containers.GenericContainer;
-
-/**
- * Use {@link GenericContainer}
- * from <a href="https://www.testcontainers.org/">Testcontainers.org</a> project
- * to handle a MongoDB docker container.
- */
-public class MongoContainer extends GenericContainer<MongoContainer> {
-
- public static final int MONGO_PORT = 27017;
-
- public MongoContainer(String version) {
- super("mongo:" + version);
- withExposedPorts(MONGO_PORT);
- }
-
- public ServerAddress getServerAddress() {
- String ipAddress = getContainerIpAddress();
- int port = getMongoPort();
- return new ServerAddress(ipAddress, port);
- }
-
- public int getMongoPort() {
- return getMappedPort(MONGO_PORT);
- }
-}
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java
index b7d2c9e..0195571 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java
@@ -17,19 +17,13 @@
*/
package org.apache.gora.mongodb.authentications;
-import com.mongodb.ServerAddress;
import org.apache.gora.GoraTestDriver;
-import org.apache.gora.mongodb.MongoContainer;
import org.apache.gora.mongodb.store.MongoStore;
import org.apache.gora.mongodb.store.MongoStoreParameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testcontainers.containers.Container;
-
-import java.io.IOException;
-import java.time.Duration;
-
-import static org.apache.commons.lang3.StringUtils.isEmpty;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
/**
* Driver to set up an embedded MongoDB database instance for use in our * unit tests.
@@ -37,54 +31,36 @@
*/
class GoraMongodbAuthenticationTestDriver extends GoraTestDriver {
private static final Logger log = LoggerFactory.getLogger(GoraMongodbAuthenticationTestDriver.class);
- private MongoContainer _container;
- private final String adminUsername = "madhawa";
- private final String adminPassword = "123";
- private final String useVersion;
+ private GenericContainer _container;
+ private static final String adminUsername = "madhawa";
+ private static final String adminPassword = "123";
private final String authMechanisms;
- GoraMongodbAuthenticationTestDriver(String authMechanisms, String useVersion) {
+ GoraMongodbAuthenticationTestDriver(String authMechanisms, GenericContainer container) {
super(MongoStore.class);
this.authMechanisms = authMechanisms;
- this.useVersion = useVersion;
+ this._container = container;
}
- private void doStart() throws Exception {
- try {
- log.info("Starting the embedded Mongodb server");
- startWithAuth();
- if (authMechanisms.equals("SCRAM-SHA-1")) {
- setSCRAM_SHA_1Credentials();
- }
- // Store Mongo server "host:port" in Hadoop configuration
- // so that MongoStore will be able to get it latter
- ServerAddress address = _container.getServerAddress();
- int port = address.getPort();
- String host = address.getHost();
+ @Override
+ public void setUpClass() throws Exception {
+ log.info("Starting the embedded Mongodb server");
- conf.set(MongoStoreParameters.PROP_MONGO_SERVERS, host + ":" + port);
- conf.set(MongoStoreParameters.PROP_MONGO_DB, "admin");
- conf.set(MongoStoreParameters.PROP_MONGO_AUTHENTICATION_TYPE, authMechanisms);
- conf.set(MongoStoreParameters.PROP_MONGO_LOGIN, adminUsername);
- conf.set(MongoStoreParameters.PROP_MONGO_SECRET, adminPassword);
- } catch (Exception e) {
- log.error("Error starting embedded Mongodb server... tearing down test driver.");
- tearDownClass();
- }
+ // Store Mongo server "host:port" in Hadoop configuration
+ // so that MongoStore will be able to get it latter
+ int port = _container.getMappedPort(27017);
+ String host = _container.getContainerIpAddress();
+
+ conf.set(MongoStoreParameters.PROP_MONGO_SERVERS, host + ":" + port);
+ conf.set(MongoStoreParameters.PROP_MONGO_DB, "admin");
+ conf.set(MongoStoreParameters.PROP_MONGO_AUTHENTICATION_TYPE, authMechanisms);
+ conf.set(MongoStoreParameters.PROP_MONGO_LOGIN, adminUsername);
+ conf.set(MongoStoreParameters.PROP_MONGO_SECRET, adminPassword);
}
- private void startWithAuth() throws IOException {
- try {
- prepareExecutable();
- _container.start();
- } catch (Exception e) {
- log.error("Error starting embedded Mongodb server... tearing down test driver.");
- tearDownClass();
- }
- }
+ public static GenericContainer mongoContainer(String authMechanisms, String useVersion) {
+ GenericContainer _container = new GenericContainer(useVersion).withExposedPorts(27017);
- private void prepareExecutable() throws IOException {
- _container = new MongoContainer(useVersion);
// https://hub.docker.com/_/mongo
// These variables, used in conjunction, create a new user and set that user's password.
// This user is created in the admin authentication database
@@ -93,43 +69,16 @@
_container.withEnv("MONGO_INITDB_ROOT_PASSWORD", adminPassword);
// To enable authentication, MongoDB will have to restart itself
- // so wait for at least 5 sec
- _container.withMinimumRunningDuration(Duration.ofSeconds(5));
+ int restartCount = 2;
+ _container.waitingFor(
+ Wait.forLogMessage("(?i).*waiting for connections.*", restartCount)
+ );
// https://docs.mongodb.com/manual/tutorial/enable-authentication/
// https://docs.mongodb.com/manual/reference/parameters/#param.authenticationMechanisms
_container.withCommand("--auth", "--setParameter", "authenticationMechanisms=" + authMechanisms);
- }
- private void setSCRAM_SHA_1Credentials() throws Exception {
- final String scriptText1 = "db.adminCommand({authSchemaUpgrade: 1});\n";
- runScriptAndWait(scriptText1, "admin", adminUsername, adminPassword);
- }
-
- private void runScriptAndWait(String scriptText, String dbName, String username, String password)
- throws InterruptedException, IOException {
- final StringBuilder builder = new StringBuilder("mongo --quiet");
- if (!isEmpty(username)) {
- builder.append(" --username ").append(username);
- }
- if (!isEmpty(password)) {
- builder.append(" --password ").append(password);
- }
- if (!isEmpty(dbName)) {
- builder.append(" ").append(dbName);
- }
- builder.append(" --eval '").append(scriptText).append("'");
-
- Container.ExecResult res = _container.execInContainer("/bin/bash", "-c", builder.toString());
- if (!isEmpty(res.getStderr())) {
- log.error("Unable to run script on Mongodb server {}: {}", scriptText, res.getStderr());
- throw new IOException(res.getStderr());
- }
- }
-
- @Override
- public void setUpClass() throws Exception {
- doStart();
+ return _container;
}
/**
@@ -137,7 +86,5 @@
*/
@Override
public void tearDownClass() {
- log.info("Shutting down mongodb server...");
- _container.stop();
}
}
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java
index a44f70d..a393817 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,20 +18,22 @@
package org.apache.gora.mongodb.authentications;
import org.apache.gora.mongodb.store.TestMongoStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.junit.ClassRule;
+import org.testcontainers.containers.GenericContainer;
+
+import static org.apache.gora.mongodb.authentications.GoraMongodbAuthenticationTestDriver.mongoContainer;
/**
* Perform {@link TestMongoStore} tests on MongoDB 3.6.x server with Plain Authentication mechanism.
*/
public class PLAIN_AuthenticationTest extends TestMongoStore {
- private static Logger log = LoggerFactory
- .getLogger(PLAIN_AuthenticationTest.class);
- static {
- try {
- setTestDriver(new GoraMongodbAuthenticationTestDriver("PLAIN", "3.6"));
- } catch (Exception e) {
- log.error("MongoDb Test Driver initialization failed. "+ e.getMessage());
+
+ public static final String AUTH_MECHANISMS = "PLAIN";
+
+ @ClassRule
+ public final static GenericContainer container = mongoContainer(AUTH_MECHANISMS, "mongo:3.6");
+
+ static {
+ setTestDriver(new GoraMongodbAuthenticationTestDriver(AUTH_MECHANISMS, container));
}
- }
}
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java
index c8e95fb..2346dfb 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,16 +18,22 @@
package org.apache.gora.mongodb.authentications;
import org.apache.gora.mongodb.store.TestMongoStore;
+import org.junit.ClassRule;
+import org.testcontainers.containers.GenericContainer;
+
+import static org.apache.gora.mongodb.authentications.GoraMongodbAuthenticationTestDriver.mongoContainer;
/**
- * Perform {@link TestMongoStore} tests on MongoDB 3.6.x server with SCRAM-SHA-1 Authentication mechanism
+ * Perform {@link TestMongoStore} tests on MongoDB 4.2.x server with SCRAM-SHA-1 Authentication mechanism
*/
public class SCRAM_SHA_1_AuthenticationTest extends TestMongoStore {
- static {
- try {
- setTestDriver(new GoraMongodbAuthenticationTestDriver("SCRAM-SHA-1", "3.6"));
- } catch (Exception e) {
- log.error("MongoDb Test Driver initialization failed. "+ e.getMessage());
+
+ public static final String AUTH_MECHANISMS = "SCRAM-SHA-1";
+
+ @ClassRule
+ public final static GenericContainer container = mongoContainer(AUTH_MECHANISMS, "mongo:4.2");
+
+ static {
+ setTestDriver(new GoraMongodbAuthenticationTestDriver(AUTH_MECHANISMS, container));
}
- }
}
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/mapreduce/GoraMongoMapredTest.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/mapreduce/GoraMongoMapredTest.java
index 7315192..9b1fe1c 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/mapreduce/GoraMongoMapredTest.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/mapreduce/GoraMongoMapredTest.java
@@ -19,10 +19,10 @@
import org.apache.gora.GoraTestDriver;
import org.apache.gora.mongodb.GoraMongodbTestDriver;
-import org.apache.gora.mongodb.MongoContainer;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.testcontainers.containers.MongoDBContainer;
/**
* Created by drazzib on 24/05/14.
@@ -30,7 +30,7 @@
public class GoraMongoMapredTest {
@ClassRule
- public final static MongoContainer container = new MongoContainer("3.6");
+ public final static MongoDBContainer container = new MongoDBContainer("mongo:3.6");
protected static GoraTestDriver testDriver = new GoraMongodbTestDriver(container);
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore34.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore34.java
index 5e54486..03c83d8 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore34.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore34.java
@@ -18,8 +18,8 @@
package org.apache.gora.mongodb.store;
import org.apache.gora.mongodb.GoraMongodbTestDriver;
-import org.apache.gora.mongodb.MongoContainer;
import org.junit.ClassRule;
+import org.testcontainers.containers.MongoDBContainer;
/**
* Perform {@link TestMongoStore} tests on MongoDB 3.4.x server.
@@ -27,7 +27,7 @@
public class TestMongoStore34 extends TestMongoStore {
@ClassRule
- public final static MongoContainer container = new MongoContainer("3.4");
+ public final static MongoDBContainer container = new MongoDBContainer("mongo:3.4");
static {
setTestDriver(new GoraMongodbTestDriver(container));
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore36.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore36.java
index dd69e42..0a4e7b5 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore36.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore36.java
@@ -18,8 +18,8 @@
package org.apache.gora.mongodb.store;
import org.apache.gora.mongodb.GoraMongodbTestDriver;
-import org.apache.gora.mongodb.MongoContainer;
import org.junit.ClassRule;
+import org.testcontainers.containers.MongoDBContainer;
/**
* Perform {@link TestMongoStore} tests on MongoDB 3.6.x server.
@@ -27,7 +27,7 @@
public class TestMongoStore36 extends TestMongoStore {
@ClassRule
- public final static MongoContainer container = new MongoContainer("3.6");
+ public final static MongoDBContainer container = new MongoDBContainer("mongo:3.6");
static {
setTestDriver(new GoraMongodbTestDriver(container));
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore40.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore40.java
index 6eb3d8b..ca94271 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore40.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore40.java
@@ -18,8 +18,8 @@
package org.apache.gora.mongodb.store;
import org.apache.gora.mongodb.GoraMongodbTestDriver;
-import org.apache.gora.mongodb.MongoContainer;
import org.junit.ClassRule;
+import org.testcontainers.containers.MongoDBContainer;
/**
* Perform {@link TestMongoStore} tests on MongoDB 4.0.x server.
@@ -27,7 +27,7 @@
public class TestMongoStore40 extends TestMongoStore {
@ClassRule
- public final static MongoContainer container = new MongoContainer("4.0");
+ public final static MongoDBContainer container = new MongoDBContainer("mongo:4.0");
static {
setTestDriver(new GoraMongodbTestDriver(container));
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore42.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore42.java
index 3d976c1..9d39589 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore42.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore42.java
@@ -18,8 +18,8 @@
package org.apache.gora.mongodb.store;
import org.apache.gora.mongodb.GoraMongodbTestDriver;
-import org.apache.gora.mongodb.MongoContainer;
import org.junit.ClassRule;
+import org.testcontainers.containers.MongoDBContainer;
/**
* Perform {@link TestMongoStore} tests on MongoDB 4.2.x server.
@@ -27,7 +27,7 @@
public class TestMongoStore42 extends TestMongoStore {
@ClassRule
- public final static MongoContainer container = new MongoContainer("4.2");
+ public final static MongoDBContainer container = new MongoDBContainer("mongo:4.2");
static {
setTestDriver(new GoraMongodbTestDriver(container));
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMappingFromProperties.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMappingFromProperties.java
index 550d25e..2a894e4 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMappingFromProperties.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMappingFromProperties.java
@@ -18,16 +18,14 @@
package org.apache.gora.mongodb.store;
-import com.mongodb.ServerAddress;
import junit.framework.Assert;
import org.apache.commons.io.IOUtils;
import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.mongodb.MongoContainer;
import org.apache.gora.store.DataStoreFactory;
import org.apache.hadoop.conf.Configuration;
-import org.junit.After;
-import org.junit.Before;
+import org.junit.ClassRule;
import org.junit.Test;
+import org.testcontainers.containers.MongoDBContainer;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -37,14 +35,9 @@
* Test case for loading mappings from properties
*/
public class TestMongoStoreMappingFromProperties {
- private MongoContainer _container;
- @Before
- public void setUp() {
- // Container for MongoStore
- this._container = new MongoContainer("4.2");
- _container.start();
- }
+ @ClassRule
+ public final static MongoDBContainer _container = new MongoDBContainer("mongo:4.2");
@Test
public void testInitialize() throws IOException {
@@ -57,9 +50,8 @@
"</gora-otd>";
// Initiate the MongoDB server on the default port
- ServerAddress address = _container.getServerAddress();
- int port = address.getPort();
- String host = address.getHost();
+ int port = _container.getFirstMappedPort();
+ String host = _container.getContainerIpAddress();
Properties prop = DataStoreFactory.createProps();
@@ -82,8 +74,4 @@
Assert.assertEquals(expectedMapping, actualMapping);
}
- @After
- public void tearDown() {
- _container.stop();
- }
}
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMetadataAnalyzer.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMetadataAnalyzer.java
index 34f280d..92f1ed9 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMetadataAnalyzer.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMetadataAnalyzer.java
@@ -21,18 +21,17 @@
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoDatabase;
import org.apache.gora.mongodb.GoraMongodbTestDriver;
-import org.apache.gora.mongodb.MongoContainer;
import org.apache.gora.store.DataStoreFactory;
import org.apache.gora.store.DataStoreMetadataFactory;
import org.apache.gora.store.impl.DataStoreMetadataAnalyzer;
import org.apache.gora.util.GoraException;
import org.apache.hadoop.conf.Configuration;
import org.bson.Document;
-import org.bson.types.ObjectId;
import org.junit.Assert;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
+import org.testcontainers.containers.MongoDBContainer;
import java.util.*;
@@ -44,7 +43,7 @@
private MongoDatabase mongoDatabase;
@ClassRule
- public final static MongoContainer container = new MongoContainer("4.2");
+ public final static MongoDBContainer container = new MongoDBContainer("mongo:4.2");
static {
setTestDriver(new GoraMongodbTestDriver(container));
diff --git a/gora-redis/src/test/java/org/apache/gora/redis/GoraRedisTestDriver.java b/gora-redis/src/test/java/org/apache/gora/redis/GoraRedisTestDriver.java
index 1180b23..ac6610d 100755
--- a/gora-redis/src/test/java/org/apache/gora/redis/GoraRedisTestDriver.java
+++ b/gora-redis/src/test/java/org/apache/gora/redis/GoraRedisTestDriver.java
@@ -17,8 +17,6 @@
*/
package org.apache.gora.redis;
-import java.io.IOException;
-import java.time.Duration;
import org.apache.gora.GoraTestDriver;
import org.apache.gora.redis.store.RedisStore;
import org.apache.gora.redis.util.RedisStartupLogWaitStrategy;
@@ -26,6 +24,9 @@
import org.apache.gora.redis.util.StorageMode;
import org.testcontainers.containers.GenericContainer;
+import java.io.IOException;
+import java.time.Duration;
+
/**
* Helper class to execute tests in a embedded instance of Redis.
*
diff --git a/gora-redis/src/test/java/org/apache/gora/redis/util/RedisStartupLogWaitStrategy.java b/gora-redis/src/test/java/org/apache/gora/redis/util/RedisStartupLogWaitStrategy.java
index b9857b1..f107c1d 100755
--- a/gora-redis/src/test/java/org/apache/gora/redis/util/RedisStartupLogWaitStrategy.java
+++ b/gora-redis/src/test/java/org/apache/gora/redis/util/RedisStartupLogWaitStrategy.java
@@ -17,38 +17,19 @@
*/
package org.apache.gora.redis.util;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.function.Predicate;
-import org.testcontainers.containers.ContainerLaunchException;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.output.OutputFrame;
-import org.testcontainers.containers.output.WaitingConsumer;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
/**
* Utility class for detecting when the docker container is ready.
*/
-public class RedisStartupLogWaitStrategy extends GenericContainer.AbstractWaitStrategy {
+public class RedisStartupLogWaitStrategy extends LogMessageWaitStrategy {
private static final String REGEX = ".*Background AOF rewrite finished successfully.*";
private final int times = 3;
- @Override
- protected void waitUntilReady() {
- WaitingConsumer waitingConsumer = new WaitingConsumer();
- this.container.followOutput(waitingConsumer);
- Predicate waitPredicate = (outputFrame) -> {
- String trimmedFrameText = ((OutputFrame) outputFrame).getUtf8String().replaceFirst("\n$", "");
- return trimmedFrameText.matches(REGEX);
- };
-
- try {
- waitingConsumer.waitUntil(waitPredicate, this.startupTimeout.getSeconds(), TimeUnit.SECONDS,
- this.times);
- } catch (TimeoutException var4) {
- throw new ContainerLaunchException(
- "Timed out waiting for log output matching Redis server startup Log \'" + REGEX
- + "\'");
- }
+ public RedisStartupLogWaitStrategy() {
+ withRegEx(REGEX);
+ withTimes(times);
}
+
}
diff --git a/gora-rethinkdb/src/test/java/org/apache/gora/rethinkdb/store/RethinkDBStartupWaitStrategy.java b/gora-rethinkdb/src/test/java/org/apache/gora/rethinkdb/store/RethinkDBStartupWaitStrategy.java
index 9c1939b..0425fcb 100644
--- a/gora-rethinkdb/src/test/java/org/apache/gora/rethinkdb/store/RethinkDBStartupWaitStrategy.java
+++ b/gora-rethinkdb/src/test/java/org/apache/gora/rethinkdb/store/RethinkDBStartupWaitStrategy.java
@@ -18,35 +18,14 @@
package org.apache.gora.rethinkdb.store;
-import org.testcontainers.containers.ContainerLaunchException;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.output.OutputFrame;
-import org.testcontainers.containers.output.WaitingConsumer;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.function.Predicate;
-
-public class RethinkDBStartupWaitStrategy extends GenericContainer.AbstractWaitStrategy {
+public class RethinkDBStartupWaitStrategy extends LogMessageWaitStrategy {
private static final String regEx = "Server ready,.*";
- private int times = 1;
-
- protected void waitUntilReady() {
- WaitingConsumer waitingConsumer = new WaitingConsumer();
- this.container.followOutput(waitingConsumer);
- Predicate waitPredicate = (outputFrame) -> {
- String trimmedFrameText = ((OutputFrame) outputFrame).getUtf8String().replaceFirst("\n$", "");
- return trimmedFrameText.matches(regEx);
- };
-
- try {
- waitingConsumer.waitUntil(waitPredicate, this.startupTimeout.getSeconds(), TimeUnit.SECONDS,
- this.times);
- } catch (TimeoutException var4) {
- throw new ContainerLaunchException(
- "Timed out waiting for log output matching RethinkDB server startup Log \'" + regEx + "\'");
- }
+ public RethinkDBStartupWaitStrategy() {
+ withRegEx(regEx);
}
+
}
\ No newline at end of file
diff --git a/gora-solr/src/test/conf/log4j.properties b/gora-solr/src/test/conf/log4j.properties
index 3094e6b..62f4caa 100644
--- a/gora-solr/src/test/conf/log4j.properties
+++ b/gora-solr/src/test/conf/log4j.properties
@@ -14,7 +14,7 @@
# limitations under the License.
# Set root logger level to error
-log4j.rootLogger=DEBUG, Console
+log4j.rootLogger=INFO, Console
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.apache.hadoop=WARN
diff --git a/pom.xml b/pom.xml
index 2f86e9d..18f2b94 100755
--- a/pom.xml
+++ b/pom.xml
@@ -622,6 +622,7 @@
<argLine>-Xmx512m</argLine>
<forkMode>always</forkMode>
<testFailureIgnore>false</testFailureIgnore>
+ <forkedProcessTimeoutInSeconds>2400</forkedProcessTimeoutInSeconds>
</configuration>
</plugin>
<plugin>
@@ -883,7 +884,7 @@
<!-- Testing Dependencies -->
<junit.version>4.10</junit.version>
- <test.container.version>1.4.2</test.container.version>
+ <test.container.version>1.14.3</test.container.version>
<!-- gora-benchmark and version dependencies -->
<site.ycsb.version>0.17.0</site.ycsb.version>
@@ -1828,8 +1829,10 @@
<dependency>
<groupId>org.testcontainers</groupId>
- <artifactId>testcontainers</artifactId>
+ <artifactId>testcontainers-bom</artifactId>
<version>${test.container.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
</dependency>
<!-- Gora Benchmark Dependencies -->
