This is an automated email from the ASF dual-hosted git repository. drazzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/gora.git
The following commit(s) were added to refs/heads/master by this push: new b45581a GORA-669 Improve tests stability on Jenkins (#233) b45581a is described below commit b45581a371d2d69c472c37793efa085436056c9b Author: Damien Raude-Morvan <draz...@apache.org> AuthorDate: Wed Nov 11 16:41:10 2020 +0100 GORA-669 Improve tests stability on Jenkins (#233) * Synchronize Maven options in Github Actions (`.github/workflows/master-pr-build.yml`) and Jenkins build (`Jenkinsfile`) * Upgrade testcontainers to 1.14.3 * Use upstream `MongoDBContainer` and remove our old `MongoContainer` implementation * Rework our `*LogWaitStrategy` to reduce boilerplate code * Improve stability of our `GoraMongodbAuthenticationTestDriver` by assuring that "waiting for connections" log message if present twice (because MongoDB docker container restart daemon itself) * Reduce log verbosity of gora-solr tests * Enforce a global `forkedProcessTimeoutInSeconds` of 40 minutes in pom.xml --- .github/workflows/master-pr-build.yml | 22 +++-- .github/workflows/master-push-build.yml | 14 ++- Jenkinsfile | 4 +- .../TestAerospikeStoreMapReduceSerialization.java | 5 +- .../store/AerospikeStartupLogWaitStrategy.java | 31 ++---- gora-benchmark/pom.xml | 6 +- .../org/apache/gora/benchmark/GoraClientTest.java | 13 +-- gora-couchdb/pom.xml | 6 -- .../couchdb/CouchDBStartupLogWaitStrategy.java | 32 +----- .../org/apache/gora/hive/store/TestHiveStore.java | 1 + gora-mongodb/pom.xml | 7 ++ .../src/test/conf/log4j.properties | 18 +--- .../apache/gora/mongodb/GoraMongodbTestDriver.java | 13 +-- .../org/apache/gora/mongodb/MongoContainer.java | 46 --------- .../GoraMongodbAuthenticationTestDriver.java | 109 ++++++--------------- .../authentications/PLAIN_AuthenticationTest.java | 28 +++--- .../SCRAM_SHA_1_AuthenticationTest.java | 26 +++-- .../mongodb/mapreduce/GoraMongoMapredTest.java | 4 +- .../gora/mongodb/store/TestMongoStore34.java | 4 +- .../gora/mongodb/store/TestMongoStore36.java | 4 +- .../gora/mongodb/store/TestMongoStore40.java | 4 +- .../gora/mongodb/store/TestMongoStore42.java | 4 +- .../store/TestMongoStoreMappingFromProperties.java | 24 ++--- .../store/TestMongoStoreMetadataAnalyzer.java | 5 +- .../org/apache/gora/redis/GoraRedisTestDriver.java | 5 +- .../redis/util/RedisStartupLogWaitStrategy.java | 31 ++---- .../store/RethinkDBStartupWaitStrategy.java | 31 +----- gora-solr/src/test/conf/log4j.properties | 2 +- pom.xml | 7 +- 29 files changed, 162 insertions(+), 344 deletions(-) diff --git a/.github/workflows/master-pr-build.yml b/.github/workflows/master-pr-build.yml index 5d2a207..5ab3b17 100644 --- a/.github/workflows/master-pr-build.yml +++ b/.github/workflows/master-pr-build.yml @@ -21,6 +21,10 @@ on: pull_request: branches: - master + +env: + MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version --no-transfer-progress" + jobs: build: runs-on: ubuntu-latest @@ -28,10 +32,14 @@ jobs: matrix: java: [ '1.8' ] steps: - - uses: actions/checkout@v1 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - name: mvn sourcecheck - run: mvn --no-transfer-progress --batch-mode clean install \ No newline at end of file + - uses: actions/checkout@v2 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Build + run: mvn $MAVEN_CLI_OPTS -DskipTests clean install + + - name: Test + run: mvn $MAVEN_CLI_OPTS verify \ No newline at end of file diff --git a/.github/workflows/master-push-build.yml b/.github/workflows/master-push-build.yml index 3e3c627..517f455 100644 --- a/.github/workflows/master-push-build.yml +++ b/.github/workflows/master-push-build.yml @@ -21,6 +21,10 @@ on: push: branches: - master + +env: + MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version --no-transfer-progress" + jobs: build: runs-on: ubuntu-latest @@ -28,10 +32,14 @@ jobs: matrix: java: [ '1.8' ] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - - name: mvn sourcecheck - run: mvn --no-transfer-progress --batch-mode clean install \ No newline at end of file + + - name: Build + run: mvn $MAVEN_CLI_OPTS -DskipTests clean install + + - name: Test + run: mvn $MAVEN_CLI_OPTS verify \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index b3eebee..cbae42f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { // only keep the latest 10 builds buildDiscarder logRotator(numToKeepStr: '10') // cancel build if not complete within two hours of scheduling - timeout time: 2, unit: 'HOURS' + timeout time: 3, unit: 'HOURS' disableConcurrentBuilds() } @@ -86,7 +86,7 @@ pipeline { post { always { - junit '**/target/surefire-reports/TEST-*.xml' + junit testResults: '**/target/surefire-reports/TEST-*.xml', testDataPublishers: [[$class: 'StabilityTestDataPublisher']] } } } diff --git a/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java index b8f281e..58c1654 100644 --- a/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java +++ b/gora-aerospike/src/test/java/org/apache/gora/aerospike/mapreduce/TestAerospikeStoreMapReduceSerialization.java @@ -22,11 +22,10 @@ import org.apache.gora.examples.generated.WebPage; import org.apache.gora.mapreduce.MapReduceTestUtils; import org.apache.gora.store.DataStoreFactory; import org.apache.hadoop.conf.Configuration; -import org.junit.Test; -import org.junit.Before; import org.junit.After; +import org.junit.Before; import org.junit.ClassRule; - +import org.junit.Test; import org.testcontainers.containers.GenericContainer; import java.time.Duration; diff --git a/gora-aerospike/src/test/java/org/apache/gora/aerospike/store/AerospikeStartupLogWaitStrategy.java b/gora-aerospike/src/test/java/org/apache/gora/aerospike/store/AerospikeStartupLogWaitStrategy.java index 8b9cf5e..4881f93 100644 --- a/gora-aerospike/src/test/java/org/apache/gora/aerospike/store/AerospikeStartupLogWaitStrategy.java +++ b/gora-aerospike/src/test/java/org/apache/gora/aerospike/store/AerospikeStartupLogWaitStrategy.java @@ -18,36 +18,17 @@ package org.apache.gora.aerospike.store; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.OutputFrame; -import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Predicate; - -public class AerospikeStartupLogWaitStrategy extends GenericContainer.AbstractWaitStrategy { +public class AerospikeStartupLogWaitStrategy extends LogMessageWaitStrategy { private static final String regEx = ".*heartbeat-received: self 0 foreign 0.*"; private int times = 2; - protected void waitUntilReady() { - WaitingConsumer waitingConsumer = new WaitingConsumer(); - this.container.followOutput(waitingConsumer); - Predicate waitPredicate = (outputFrame) -> { - String trimmedFrameText = ((OutputFrame) outputFrame).getUtf8String().replaceFirst("\n$", ""); - return trimmedFrameText.matches(regEx); - }; - - try { - waitingConsumer.waitUntil(waitPredicate, this.startupTimeout.getSeconds(), TimeUnit.SECONDS, - this.times); - } catch (TimeoutException var4) { - throw new ContainerLaunchException( - "Timed out waiting for log output matching Aerospike server startup Log \'" + regEx - + "\'"); - } + public AerospikeStartupLogWaitStrategy() { + withRegEx(regEx); + withTimes(times); } + } \ No newline at end of file diff --git a/gora-benchmark/pom.xml b/gora-benchmark/pom.xml index c8d6e9d..811cd68 100644 --- a/gora-benchmark/pom.xml +++ b/gora-benchmark/pom.xml @@ -188,9 +188,8 @@ </dependency> <dependency> - <groupId>org.apache.gora</groupId> - <artifactId>gora-mongodb</artifactId> - <type>test-jar</type> + <groupId>org.testcontainers</groupId> + <artifactId>mongodb</artifactId> <scope>test</scope> </dependency> @@ -199,5 +198,6 @@ <groupId>org.ektorp</groupId> <artifactId>org.ektorp</artifactId> </dependency> + </dependencies> </project> diff --git a/gora-benchmark/src/test/java/org/apache/gora/benchmark/GoraClientTest.java b/gora-benchmark/src/test/java/org/apache/gora/benchmark/GoraClientTest.java index 91a7c07..7ce0127 100644 --- a/gora-benchmark/src/test/java/org/apache/gora/benchmark/GoraClientTest.java +++ b/gora-benchmark/src/test/java/org/apache/gora/benchmark/GoraClientTest.java @@ -17,9 +17,7 @@ */ package org.apache.gora.benchmark; -import com.mongodb.ServerAddress; import org.apache.gora.benchmark.generated.User; -import org.apache.gora.mongodb.MongoContainer; import org.apache.gora.store.DataStoreFactory; import org.apache.gora.util.GoraException; import org.junit.After; @@ -27,6 +25,7 @@ import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.containers.MongoDBContainer; import site.ycsb.ByteIterator; import site.ycsb.Status; import site.ycsb.StringByteIterator; @@ -50,7 +49,7 @@ public class GoraClientTest { private static HashMap<String, ByteIterator> DATA_TO_UPDATE; private static HashMap<String, ByteIterator> INTEGER_DATA; private static boolean isMongoDBSetupDone = false; - private static MongoContainer mongo; + private static MongoDBContainer mongo; /** * Setup MongoDB embed cluster. This function will auto provision a MongoDB @@ -60,11 +59,10 @@ public class GoraClientTest { private void setupMongoDBCluster() { try { if (!isMongoDBSetupDone) { - mongo = new MongoContainer("3.6"); + mongo = new MongoDBContainer("mongo:3.6"); mongo.start(); } - ServerAddress address = mongo.getServerAddress(); - LOG.info("Started MongoDB Server on " + address.getHost() + ":" + address.getPort()); + LOG.info("Started MongoDB Server on " + mongo.getReplicaSetUrl()); isMongoDBSetupDone = true; } catch (Exception e) { LOG.info("Cannot Start MongoDB Server {}", e.getMessage(), e); @@ -99,8 +97,7 @@ public class GoraClientTest { String dataStoreToTest = GoraBenchmarkUtils.getDataBase(dataStoreProperties); if (Constants.MONGODB.equals(dataStoreToTest)) { setupMongoDBCluster(); - ServerAddress address = mongo.getServerAddress(); - properties.setProperty("gora.mongodb.servers", address.getHost() + ":" + address.getPort()); + properties.setProperty("gora.mongodb.servers", mongo.getContainerIpAddress() + ":" + mongo.getFirstMappedPort()); } benchmarkClient = new GoraBenchmarkClient(); diff --git a/gora-couchdb/pom.xml b/gora-couchdb/pom.xml index 4ff0b93..e1add43 100644 --- a/gora-couchdb/pom.xml +++ b/gora-couchdb/pom.xml @@ -154,12 +154,6 @@ </dependency> <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-simple</artifactId> - <scope>compile</scope> - </dependency> - - <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <scope>runtime</scope> diff --git a/gora-couchdb/src/test/java/org/apache/gora/couchdb/CouchDBStartupLogWaitStrategy.java b/gora-couchdb/src/test/java/org/apache/gora/couchdb/CouchDBStartupLogWaitStrategy.java index 7cd06cf..6fc043d 100644 --- a/gora-couchdb/src/test/java/org/apache/gora/couchdb/CouchDBStartupLogWaitStrategy.java +++ b/gora-couchdb/src/test/java/org/apache/gora/couchdb/CouchDBStartupLogWaitStrategy.java @@ -18,43 +18,19 @@ package org.apache.gora.couchdb; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.OutputFrame; -import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Predicate; /** * Log based CouchDB server startup wait strategy to sync server * startup to test suit startup. */ -public class CouchDBStartupLogWaitStrategy extends GenericContainer.AbstractWaitStrategy { +public class CouchDBStartupLogWaitStrategy extends LogMessageWaitStrategy { private static final String regEx = ".*Apache CouchDB has started. Time to relax..*"; - private int times = 1; - - protected void waitUntilReady() { - WaitingConsumer waitingConsumer = new WaitingConsumer(); - this.container.followOutput(waitingConsumer); - Predicate waitPredicate = (outputFrame) -> { - String trimmedFrameText = ((OutputFrame) outputFrame) - .getUtf8String() - .replaceFirst("\n$", ""); - return trimmedFrameText.matches(regEx); - }; - - try { - waitingConsumer.waitUntil(waitPredicate, this.startupTimeout.getSeconds(), TimeUnit.SECONDS, - this.times); - } catch (TimeoutException var4) { - throw new ContainerLaunchException( - "Timed out waiting for log output matching CouchDB server startup Log \'" + regEx - + "\'"); - } + public CouchDBStartupLogWaitStrategy() { + withRegEx(regEx); } } \ No newline at end of file diff --git a/gora-hive/src/test/java/org/apache/gora/hive/store/TestHiveStore.java b/gora-hive/src/test/java/org/apache/gora/hive/store/TestHiveStore.java index 0d635f2..003d8ec 100644 --- a/gora-hive/src/test/java/org/apache/gora/hive/store/TestHiveStore.java +++ b/gora-hive/src/test/java/org/apache/gora/hive/store/TestHiveStore.java @@ -35,6 +35,7 @@ import org.apache.gora.store.DataStoreTestUtil; import org.apache.gora.util.GoraException; import org.apache.gora.util.StringUtils; import org.junit.Ignore; +import org.junit.Test; /** * HiveStore Tests extending {@link DataStoreTestBase} which run the base JUnit test suite for diff --git a/gora-mongodb/pom.xml b/gora-mongodb/pom.xml index e802e0f..bd43ae5 100644 --- a/gora-mongodb/pom.xml +++ b/gora-mongodb/pom.xml @@ -193,6 +193,13 @@ <artifactId>testcontainers</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>mongodb</artifactId> + <scope>test</scope> + </dependency> + <!-- END of Testing Dependencies --> </dependencies> diff --git a/gora-solr/src/test/conf/log4j.properties b/gora-mongodb/src/test/conf/log4j.properties similarity index 64% copy from gora-solr/src/test/conf/log4j.properties copy to gora-mongodb/src/test/conf/log4j.properties index 3094e6b..0256aa9 100644 --- a/gora-solr/src/test/conf/log4j.properties +++ b/gora-mongodb/src/test/conf/log4j.properties @@ -13,23 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Set root logger level to error -log4j.rootLogger=DEBUG, Console +log4j.rootLogger=INFO, Console -log4j.logger.org.apache.zookeeper=WARN -log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.testcontainers=INFO +log4j.logger.com.github.dockerjava=WARN -###### Console appender definition ####### - -# All outputs currently set to be a ConsoleAppender. log4j.appender.Console=org.apache.log4j.ConsoleAppender log4j.appender.Console.layout=org.apache.log4j.PatternLayout log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %x %-5p [%c{3}] [%t] %m%n - -###### File appender definition ####### -log4j.appender.File=org.apache.log4j.DailyRollingFileAppender -log4j.appender.File.Append=true -log4j.appender.File.DatePattern='.'yyyy-MM-dd -log4j.appender.File.layout=org.apache.log4j.PatternLayout -log4j.appender.File.layout.ConversionPattern=%d{ABSOLUTE} %-5p [%c] %m%n + diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/GoraMongodbTestDriver.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/GoraMongodbTestDriver.java index 73a60bb..f54b0bb 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/GoraMongodbTestDriver.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/GoraMongodbTestDriver.java @@ -18,12 +18,12 @@ package org.apache.gora.mongodb; import com.mongodb.MongoClient; -import com.mongodb.ServerAddress; import org.apache.gora.GoraTestDriver; import org.apache.gora.mongodb.store.MongoStore; import org.apache.gora.mongodb.store.MongoStoreParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.containers.MongoDBContainer; /** * Driver to set up an embedded MongoDB database instance for use in our @@ -34,13 +34,13 @@ public class GoraMongodbTestDriver extends GoraTestDriver { private static Logger log = LoggerFactory .getLogger(GoraMongodbTestDriver.class); - private MongoContainer _container; + private MongoDBContainer _container; private MongoClient _mongo; /** * Constructor for this class. */ - public GoraMongodbTestDriver(MongoContainer startedContainer) { + public GoraMongodbTestDriver(MongoDBContainer startedContainer) { super(MongoStore.class); this._container = startedContainer; } @@ -50,9 +50,8 @@ public class GoraMongodbTestDriver extends GoraTestDriver { */ @Override public void setUpClass() { - ServerAddress address = _container.getServerAddress(); - int port = address.getPort(); - String host = address.getHost(); + int port = _container.getMappedPort(27017); + String host = _container.getContainerIpAddress(); // Store Mongo server "host:port" in Hadoop configuration // so that MongoStore will be able to get it latter @@ -64,8 +63,6 @@ public class GoraMongodbTestDriver extends GoraTestDriver { */ @Override public void tearDownClass() { - log.info("Shutting down mongodb server..."); - _container.stop(); } } diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/MongoContainer.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/MongoContainer.java deleted file mode 100644 index 231d82c..0000000 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/MongoContainer.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gora.mongodb; - -import com.mongodb.ServerAddress; -import org.testcontainers.containers.GenericContainer; - -/** - * Use {@link GenericContainer} - * from <a href="https://www.testcontainers.org/">Testcontainers.org</a> project - * to handle a MongoDB docker container. - */ -public class MongoContainer extends GenericContainer<MongoContainer> { - - public static final int MONGO_PORT = 27017; - - public MongoContainer(String version) { - super("mongo:" + version); - withExposedPorts(MONGO_PORT); - } - - public ServerAddress getServerAddress() { - String ipAddress = getContainerIpAddress(); - int port = getMongoPort(); - return new ServerAddress(ipAddress, port); - } - - public int getMongoPort() { - return getMappedPort(MONGO_PORT); - } -} diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java index b7d2c9e..0195571 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java @@ -17,19 +17,13 @@ */ package org.apache.gora.mongodb.authentications; -import com.mongodb.ServerAddress; import org.apache.gora.GoraTestDriver; -import org.apache.gora.mongodb.MongoContainer; import org.apache.gora.mongodb.store.MongoStore; import org.apache.gora.mongodb.store.MongoStoreParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.Container; - -import java.io.IOException; -import java.time.Duration; - -import static org.apache.commons.lang3.StringUtils.isEmpty; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; /** * Driver to set up an embedded MongoDB database instance for use in our * unit tests. @@ -37,54 +31,36 @@ import static org.apache.commons.lang3.StringUtils.isEmpty; */ class GoraMongodbAuthenticationTestDriver extends GoraTestDriver { private static final Logger log = LoggerFactory.getLogger(GoraMongodbAuthenticationTestDriver.class); - private MongoContainer _container; - private final String adminUsername = "madhawa"; - private final String adminPassword = "123"; - private final String useVersion; + private GenericContainer _container; + private static final String adminUsername = "madhawa"; + private static final String adminPassword = "123"; private final String authMechanisms; - GoraMongodbAuthenticationTestDriver(String authMechanisms, String useVersion) { + GoraMongodbAuthenticationTestDriver(String authMechanisms, GenericContainer container) { super(MongoStore.class); this.authMechanisms = authMechanisms; - this.useVersion = useVersion; + this._container = container; } - private void doStart() throws Exception { - try { - log.info("Starting the embedded Mongodb server"); - startWithAuth(); - if (authMechanisms.equals("SCRAM-SHA-1")) { - setSCRAM_SHA_1Credentials(); - } - // Store Mongo server "host:port" in Hadoop configuration - // so that MongoStore will be able to get it latter - ServerAddress address = _container.getServerAddress(); - int port = address.getPort(); - String host = address.getHost(); - - conf.set(MongoStoreParameters.PROP_MONGO_SERVERS, host + ":" + port); - conf.set(MongoStoreParameters.PROP_MONGO_DB, "admin"); - conf.set(MongoStoreParameters.PROP_MONGO_AUTHENTICATION_TYPE, authMechanisms); - conf.set(MongoStoreParameters.PROP_MONGO_LOGIN, adminUsername); - conf.set(MongoStoreParameters.PROP_MONGO_SECRET, adminPassword); - } catch (Exception e) { - log.error("Error starting embedded Mongodb server... tearing down test driver."); - tearDownClass(); - } + @Override + public void setUpClass() throws Exception { + log.info("Starting the embedded Mongodb server"); + + // Store Mongo server "host:port" in Hadoop configuration + // so that MongoStore will be able to get it latter + int port = _container.getMappedPort(27017); + String host = _container.getContainerIpAddress(); + + conf.set(MongoStoreParameters.PROP_MONGO_SERVERS, host + ":" + port); + conf.set(MongoStoreParameters.PROP_MONGO_DB, "admin"); + conf.set(MongoStoreParameters.PROP_MONGO_AUTHENTICATION_TYPE, authMechanisms); + conf.set(MongoStoreParameters.PROP_MONGO_LOGIN, adminUsername); + conf.set(MongoStoreParameters.PROP_MONGO_SECRET, adminPassword); } - private void startWithAuth() throws IOException { - try { - prepareExecutable(); - _container.start(); - } catch (Exception e) { - log.error("Error starting embedded Mongodb server... tearing down test driver."); - tearDownClass(); - } - } + public static GenericContainer mongoContainer(String authMechanisms, String useVersion) { + GenericContainer _container = new GenericContainer(useVersion).withExposedPorts(27017); - private void prepareExecutable() throws IOException { - _container = new MongoContainer(useVersion); // https://hub.docker.com/_/mongo // These variables, used in conjunction, create a new user and set that user's password. // This user is created in the admin authentication database @@ -93,43 +69,16 @@ class GoraMongodbAuthenticationTestDriver extends GoraTestDriver { _container.withEnv("MONGO_INITDB_ROOT_PASSWORD", adminPassword); // To enable authentication, MongoDB will have to restart itself - // so wait for at least 5 sec - _container.withMinimumRunningDuration(Duration.ofSeconds(5)); + int restartCount = 2; + _container.waitingFor( + Wait.forLogMessage("(?i).*waiting for connections.*", restartCount) + ); // https://docs.mongodb.com/manual/tutorial/enable-authentication/ // https://docs.mongodb.com/manual/reference/parameters/#param.authenticationMechanisms _container.withCommand("--auth", "--setParameter", "authenticationMechanisms=" + authMechanisms); - } - private void setSCRAM_SHA_1Credentials() throws Exception { - final String scriptText1 = "db.adminCommand({authSchemaUpgrade: 1});\n"; - runScriptAndWait(scriptText1, "admin", adminUsername, adminPassword); - } - - private void runScriptAndWait(String scriptText, String dbName, String username, String password) - throws InterruptedException, IOException { - final StringBuilder builder = new StringBuilder("mongo --quiet"); - if (!isEmpty(username)) { - builder.append(" --username ").append(username); - } - if (!isEmpty(password)) { - builder.append(" --password ").append(password); - } - if (!isEmpty(dbName)) { - builder.append(" ").append(dbName); - } - builder.append(" --eval '").append(scriptText).append("'"); - - Container.ExecResult res = _container.execInContainer("/bin/bash", "-c", builder.toString()); - if (!isEmpty(res.getStderr())) { - log.error("Unable to run script on Mongodb server {}: {}", scriptText, res.getStderr()); - throw new IOException(res.getStderr()); - } - } - - @Override - public void setUpClass() throws Exception { - doStart(); + return _container; } /** @@ -137,7 +86,5 @@ class GoraMongodbAuthenticationTestDriver extends GoraTestDriver { */ @Override public void tearDownClass() { - log.info("Shutting down mongodb server..."); - _container.stop(); } } diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java index a44f70d..a393817 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,20 +18,22 @@ package org.apache.gora.mongodb.authentications; import org.apache.gora.mongodb.store.TestMongoStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.junit.ClassRule; +import org.testcontainers.containers.GenericContainer; + +import static org.apache.gora.mongodb.authentications.GoraMongodbAuthenticationTestDriver.mongoContainer; /** * Perform {@link TestMongoStore} tests on MongoDB 3.6.x server with Plain Authentication mechanism. */ public class PLAIN_AuthenticationTest extends TestMongoStore { - private static Logger log = LoggerFactory - .getLogger(PLAIN_AuthenticationTest.class); - static { - try { - setTestDriver(new GoraMongodbAuthenticationTestDriver("PLAIN", "3.6")); - } catch (Exception e) { - log.error("MongoDb Test Driver initialization failed. "+ e.getMessage()); + + public static final String AUTH_MECHANISMS = "PLAIN"; + + @ClassRule + public final static GenericContainer container = mongoContainer(AUTH_MECHANISMS, "mongo:3.6"); + + static { + setTestDriver(new GoraMongodbAuthenticationTestDriver(AUTH_MECHANISMS, container)); } - } } diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java index c8e95fb..2346dfb 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,16 +18,22 @@ package org.apache.gora.mongodb.authentications; import org.apache.gora.mongodb.store.TestMongoStore; +import org.junit.ClassRule; +import org.testcontainers.containers.GenericContainer; + +import static org.apache.gora.mongodb.authentications.GoraMongodbAuthenticationTestDriver.mongoContainer; /** - * Perform {@link TestMongoStore} tests on MongoDB 3.6.x server with SCRAM-SHA-1 Authentication mechanism + * Perform {@link TestMongoStore} tests on MongoDB 4.2.x server with SCRAM-SHA-1 Authentication mechanism */ public class SCRAM_SHA_1_AuthenticationTest extends TestMongoStore { - static { - try { - setTestDriver(new GoraMongodbAuthenticationTestDriver("SCRAM-SHA-1", "3.6")); - } catch (Exception e) { - log.error("MongoDb Test Driver initialization failed. "+ e.getMessage()); + + public static final String AUTH_MECHANISMS = "SCRAM-SHA-1"; + + @ClassRule + public final static GenericContainer container = mongoContainer(AUTH_MECHANISMS, "mongo:4.2"); + + static { + setTestDriver(new GoraMongodbAuthenticationTestDriver(AUTH_MECHANISMS, container)); } - } } diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/mapreduce/GoraMongoMapredTest.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/mapreduce/GoraMongoMapredTest.java index 7315192..9b1fe1c 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/mapreduce/GoraMongoMapredTest.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/mapreduce/GoraMongoMapredTest.java @@ -19,10 +19,10 @@ package org.apache.gora.mongodb.mapreduce; import org.apache.gora.GoraTestDriver; import org.apache.gora.mongodb.GoraMongodbTestDriver; -import org.apache.gora.mongodb.MongoContainer; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.testcontainers.containers.MongoDBContainer; /** * Created by drazzib on 24/05/14. @@ -30,7 +30,7 @@ import org.junit.ClassRule; public class GoraMongoMapredTest { @ClassRule - public final static MongoContainer container = new MongoContainer("3.6"); + public final static MongoDBContainer container = new MongoDBContainer("mongo:3.6"); protected static GoraTestDriver testDriver = new GoraMongodbTestDriver(container); diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore34.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore34.java index 5e54486..03c83d8 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore34.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore34.java @@ -18,8 +18,8 @@ package org.apache.gora.mongodb.store; import org.apache.gora.mongodb.GoraMongodbTestDriver; -import org.apache.gora.mongodb.MongoContainer; import org.junit.ClassRule; +import org.testcontainers.containers.MongoDBContainer; /** * Perform {@link TestMongoStore} tests on MongoDB 3.4.x server. @@ -27,7 +27,7 @@ import org.junit.ClassRule; public class TestMongoStore34 extends TestMongoStore { @ClassRule - public final static MongoContainer container = new MongoContainer("3.4"); + public final static MongoDBContainer container = new MongoDBContainer("mongo:3.4"); static { setTestDriver(new GoraMongodbTestDriver(container)); diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore36.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore36.java index dd69e42..0a4e7b5 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore36.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore36.java @@ -18,8 +18,8 @@ package org.apache.gora.mongodb.store; import org.apache.gora.mongodb.GoraMongodbTestDriver; -import org.apache.gora.mongodb.MongoContainer; import org.junit.ClassRule; +import org.testcontainers.containers.MongoDBContainer; /** * Perform {@link TestMongoStore} tests on MongoDB 3.6.x server. @@ -27,7 +27,7 @@ import org.junit.ClassRule; public class TestMongoStore36 extends TestMongoStore { @ClassRule - public final static MongoContainer container = new MongoContainer("3.6"); + public final static MongoDBContainer container = new MongoDBContainer("mongo:3.6"); static { setTestDriver(new GoraMongodbTestDriver(container)); diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore40.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore40.java index 6eb3d8b..ca94271 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore40.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore40.java @@ -18,8 +18,8 @@ package org.apache.gora.mongodb.store; import org.apache.gora.mongodb.GoraMongodbTestDriver; -import org.apache.gora.mongodb.MongoContainer; import org.junit.ClassRule; +import org.testcontainers.containers.MongoDBContainer; /** * Perform {@link TestMongoStore} tests on MongoDB 4.0.x server. @@ -27,7 +27,7 @@ import org.junit.ClassRule; public class TestMongoStore40 extends TestMongoStore { @ClassRule - public final static MongoContainer container = new MongoContainer("4.0"); + public final static MongoDBContainer container = new MongoDBContainer("mongo:4.0"); static { setTestDriver(new GoraMongodbTestDriver(container)); diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore42.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore42.java index 3d976c1..9d39589 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore42.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore42.java @@ -18,8 +18,8 @@ package org.apache.gora.mongodb.store; import org.apache.gora.mongodb.GoraMongodbTestDriver; -import org.apache.gora.mongodb.MongoContainer; import org.junit.ClassRule; +import org.testcontainers.containers.MongoDBContainer; /** * Perform {@link TestMongoStore} tests on MongoDB 4.2.x server. @@ -27,7 +27,7 @@ import org.junit.ClassRule; public class TestMongoStore42 extends TestMongoStore { @ClassRule - public final static MongoContainer container = new MongoContainer("4.2"); + public final static MongoDBContainer container = new MongoDBContainer("mongo:4.2"); static { setTestDriver(new GoraMongodbTestDriver(container)); diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMappingFromProperties.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMappingFromProperties.java index 550d25e..2a894e4 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMappingFromProperties.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMappingFromProperties.java @@ -18,16 +18,14 @@ package org.apache.gora.mongodb.store; -import com.mongodb.ServerAddress; import junit.framework.Assert; import org.apache.commons.io.IOUtils; import org.apache.gora.examples.generated.Employee; -import org.apache.gora.mongodb.MongoContainer; import org.apache.gora.store.DataStoreFactory; import org.apache.hadoop.conf.Configuration; -import org.junit.After; -import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; +import org.testcontainers.containers.MongoDBContainer; import java.io.IOException; import java.nio.charset.Charset; @@ -37,14 +35,9 @@ import java.util.Properties; * Test case for loading mappings from properties */ public class TestMongoStoreMappingFromProperties { - private MongoContainer _container; - @Before - public void setUp() { - // Container for MongoStore - this._container = new MongoContainer("4.2"); - _container.start(); - } + @ClassRule + public final static MongoDBContainer _container = new MongoDBContainer("mongo:4.2"); @Test public void testInitialize() throws IOException { @@ -57,9 +50,8 @@ public class TestMongoStoreMappingFromProperties { "</gora-otd>"; // Initiate the MongoDB server on the default port - ServerAddress address = _container.getServerAddress(); - int port = address.getPort(); - String host = address.getHost(); + int port = _container.getFirstMappedPort(); + String host = _container.getContainerIpAddress(); Properties prop = DataStoreFactory.createProps(); @@ -82,8 +74,4 @@ public class TestMongoStoreMappingFromProperties { Assert.assertEquals(expectedMapping, actualMapping); } - @After - public void tearDown() { - _container.stop(); - } } diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMetadataAnalyzer.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMetadataAnalyzer.java index 34f280d..92f1ed9 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMetadataAnalyzer.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStoreMetadataAnalyzer.java @@ -21,18 +21,17 @@ import com.google.common.collect.Sets; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoDatabase; import org.apache.gora.mongodb.GoraMongodbTestDriver; -import org.apache.gora.mongodb.MongoContainer; import org.apache.gora.store.DataStoreFactory; import org.apache.gora.store.DataStoreMetadataFactory; import org.apache.gora.store.impl.DataStoreMetadataAnalyzer; import org.apache.gora.util.GoraException; import org.apache.hadoop.conf.Configuration; import org.bson.Document; -import org.bson.types.ObjectId; import org.junit.Assert; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; +import org.testcontainers.containers.MongoDBContainer; import java.util.*; @@ -44,7 +43,7 @@ public class TestMongoStoreMetadataAnalyzer extends TestMongoStore { private MongoDatabase mongoDatabase; @ClassRule - public final static MongoContainer container = new MongoContainer("4.2"); + public final static MongoDBContainer container = new MongoDBContainer("mongo:4.2"); static { setTestDriver(new GoraMongodbTestDriver(container)); diff --git a/gora-redis/src/test/java/org/apache/gora/redis/GoraRedisTestDriver.java b/gora-redis/src/test/java/org/apache/gora/redis/GoraRedisTestDriver.java index 1180b23..ac6610d 100755 --- a/gora-redis/src/test/java/org/apache/gora/redis/GoraRedisTestDriver.java +++ b/gora-redis/src/test/java/org/apache/gora/redis/GoraRedisTestDriver.java @@ -17,8 +17,6 @@ */ package org.apache.gora.redis; -import java.io.IOException; -import java.time.Duration; import org.apache.gora.GoraTestDriver; import org.apache.gora.redis.store.RedisStore; import org.apache.gora.redis.util.RedisStartupLogWaitStrategy; @@ -26,6 +24,9 @@ import org.apache.gora.redis.util.ServerMode; import org.apache.gora.redis.util.StorageMode; import org.testcontainers.containers.GenericContainer; +import java.io.IOException; +import java.time.Duration; + /** * Helper class to execute tests in a embedded instance of Redis. * diff --git a/gora-redis/src/test/java/org/apache/gora/redis/util/RedisStartupLogWaitStrategy.java b/gora-redis/src/test/java/org/apache/gora/redis/util/RedisStartupLogWaitStrategy.java index b9857b1..f107c1d 100755 --- a/gora-redis/src/test/java/org/apache/gora/redis/util/RedisStartupLogWaitStrategy.java +++ b/gora-redis/src/test/java/org/apache/gora/redis/util/RedisStartupLogWaitStrategy.java @@ -17,38 +17,19 @@ */ package org.apache.gora.redis.util; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Predicate; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.OutputFrame; -import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; /** * Utility class for detecting when the docker container is ready. */ -public class RedisStartupLogWaitStrategy extends GenericContainer.AbstractWaitStrategy { +public class RedisStartupLogWaitStrategy extends LogMessageWaitStrategy { private static final String REGEX = ".*Background AOF rewrite finished successfully.*"; private final int times = 3; - @Override - protected void waitUntilReady() { - WaitingConsumer waitingConsumer = new WaitingConsumer(); - this.container.followOutput(waitingConsumer); - Predicate waitPredicate = (outputFrame) -> { - String trimmedFrameText = ((OutputFrame) outputFrame).getUtf8String().replaceFirst("\n$", ""); - return trimmedFrameText.matches(REGEX); - }; - - try { - waitingConsumer.waitUntil(waitPredicate, this.startupTimeout.getSeconds(), TimeUnit.SECONDS, - this.times); - } catch (TimeoutException var4) { - throw new ContainerLaunchException( - "Timed out waiting for log output matching Redis server startup Log \'" + REGEX - + "\'"); - } + public RedisStartupLogWaitStrategy() { + withRegEx(REGEX); + withTimes(times); } + } diff --git a/gora-rethinkdb/src/test/java/org/apache/gora/rethinkdb/store/RethinkDBStartupWaitStrategy.java b/gora-rethinkdb/src/test/java/org/apache/gora/rethinkdb/store/RethinkDBStartupWaitStrategy.java index 9c1939b..0425fcb 100644 --- a/gora-rethinkdb/src/test/java/org/apache/gora/rethinkdb/store/RethinkDBStartupWaitStrategy.java +++ b/gora-rethinkdb/src/test/java/org/apache/gora/rethinkdb/store/RethinkDBStartupWaitStrategy.java @@ -18,35 +18,14 @@ package org.apache.gora.rethinkdb.store; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.OutputFrame; -import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Predicate; - -public class RethinkDBStartupWaitStrategy extends GenericContainer.AbstractWaitStrategy { +public class RethinkDBStartupWaitStrategy extends LogMessageWaitStrategy { private static final String regEx = "Server ready,.*"; - private int times = 1; - - protected void waitUntilReady() { - WaitingConsumer waitingConsumer = new WaitingConsumer(); - this.container.followOutput(waitingConsumer); - Predicate waitPredicate = (outputFrame) -> { - String trimmedFrameText = ((OutputFrame) outputFrame).getUtf8String().replaceFirst("\n$", ""); - return trimmedFrameText.matches(regEx); - }; - - try { - waitingConsumer.waitUntil(waitPredicate, this.startupTimeout.getSeconds(), TimeUnit.SECONDS, - this.times); - } catch (TimeoutException var4) { - throw new ContainerLaunchException( - "Timed out waiting for log output matching RethinkDB server startup Log \'" + regEx + "\'"); - } + public RethinkDBStartupWaitStrategy() { + withRegEx(regEx); } + } \ No newline at end of file diff --git a/gora-solr/src/test/conf/log4j.properties b/gora-solr/src/test/conf/log4j.properties index 3094e6b..62f4caa 100644 --- a/gora-solr/src/test/conf/log4j.properties +++ b/gora-solr/src/test/conf/log4j.properties @@ -14,7 +14,7 @@ # limitations under the License. # Set root logger level to error -log4j.rootLogger=DEBUG, Console +log4j.rootLogger=INFO, Console log4j.logger.org.apache.zookeeper=WARN log4j.logger.org.apache.hadoop=WARN diff --git a/pom.xml b/pom.xml index 2f86e9d..18f2b94 100755 --- a/pom.xml +++ b/pom.xml @@ -622,6 +622,7 @@ <argLine>-Xmx512m</argLine> <forkMode>always</forkMode> <testFailureIgnore>false</testFailureIgnore> + <forkedProcessTimeoutInSeconds>2400</forkedProcessTimeoutInSeconds> </configuration> </plugin> <plugin> @@ -883,7 +884,7 @@ <!-- Testing Dependencies --> <junit.version>4.10</junit.version> - <test.container.version>1.4.2</test.container.version> + <test.container.version>1.14.3</test.container.version> <!-- gora-benchmark and version dependencies --> <site.ycsb.version>0.17.0</site.ycsb.version> @@ -1828,8 +1829,10 @@ <dependency> <groupId>org.testcontainers</groupId> - <artifactId>testcontainers</artifactId> + <artifactId>testcontainers-bom</artifactId> <version>${test.container.version}</version> + <type>pom</type> + <scope>import</scope> </dependency> <!-- Gora Benchmark Dependencies -->