This is an automated email from the ASF dual-hosted git repository. johncasey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 0d804d102b2 Use credential for mongodb performance tests (#25631) 0d804d102b2 is described below commit 0d804d102b2f4c9e824f7d62da3b12d037a0e806 Author: Yi Hu <ya...@google.com> AuthorDate: Tue Mar 7 14:47:55 2023 -0500 Use credential for mongodb performance tests (#25631) * Use credential for mongodb performance tests * Set cron job freq to twice a day as for other performance tests --- .../job_PerformanceTests_MongoDBIO_IT.groovy | 4 ++- ...ob_PostCommit_Python_MongoDBIO_Load_Test.groovy | 4 +-- .../kubernetes/mongodb/load-balancer/mongo.yml | 5 ++++ .../apache/beam/sdk/io/mongodb/MongoDBIOIT.java | 34 ++++++++++++++++++++-- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy b/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy index c9bc59eb858..73ef6c4a53e 100644 --- a/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy +++ b/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy @@ -24,7 +24,7 @@ String jobName = "beam_PerformanceTests_MongoDBIO_IT" job(jobName) { common.setTopLevelMainJobProperties(delegate) - common.setAutoJob(delegate,'H H/6 * * *') + common.setAutoJob(delegate,'H H/12 * * *') common.enablePhraseTriggeringFromPullRequest( delegate, 'Java MongoDBIO Performance Test', @@ -51,6 +51,8 @@ job(jobName) { mongoDBDatabaseName : 'beam', mongoDBHostName : "\$${mongoHostName}", mongoDBPort : 27017, + mongoDBUsername : 'root', + mongoDBPassword : 'uuinkkS', runner : 'DataflowRunner', autoscalingAlgorithm: 'NONE', numWorkers : '5' diff --git a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy index bee2b8f6e25..9a66b51e542 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy @@ -22,7 +22,7 @@ String jobName = "beam_python_mongoio_load_test" job(jobName) { common.setTopLevelMainJobProperties(delegate) - common.setAutoJob(delegate, 'H H/6 * * *') + common.setAutoJob(delegate, 'H H/12 * * *') common.enablePhraseTriggeringFromPullRequest( delegate, 'Python MongoDBIO Load Test', @@ -40,7 +40,7 @@ job(jobName) { temp_location: 'gs://temp-storage-for-perf-tests/loadtests', project : 'apache-beam-testing', region : 'us-central1', - mongo_uri : "mongodb://\$${mongoHostName}:27017", + mongo_uri : "mongodb://root:uuinkkS@\$${mongoHostName}:27017", num_documents: '1000000', batch_size : '10000', runner : 'DataflowRunner', diff --git a/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml b/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml index 70e19651161..19d0cf134a3 100644 --- a/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml +++ b/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml @@ -45,5 +45,10 @@ spec: containers: - name: mongo image: mongo + env: + - name: MONGO_INITDB_ROOT_USERNAME + value: root + - name: MONGO_INITDB_ROOT_PASSWORD + value: uuinkkS ports: - containerPort: 27017 diff --git a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java index b585cca7688..1294d04b44b 100644 --- a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java +++ b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.function.Function; +import org.apache.beam.repackaged.core.org.apache.commons.lang3.StringUtils; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.io.common.HashingFn; @@ -108,6 +109,20 @@ public class MongoDBIOIT { String getMongoDBDatabaseName(); void setMongoDBDatabaseName(String name); + + @Description("Username for mongodb server") + @Default.String("") + String getMongoDBUsername(); + + void setMongoDBUsername(String name); + + // Note that passwords are not as secure an authentication as other methods, and used here for + // a test environment only. + @Description("Password for mongodb server") + @Default.String("") + String getMongoDBPassword(); + + void setMongoDBPassword(String value); } private static final Map<Integer, String> EXPECTED_HASHES = @@ -127,8 +142,23 @@ public class MongoDBIOIT { PipelineOptionsFactory.register(MongoDBPipelineOptions.class); options = TestPipeline.testingPipelineOptions().as(MongoDBPipelineOptions.class); collection = String.format("test_%s", new Date().getTime()); - mongoUrl = - String.format("mongodb://%s:%s", options.getMongoDBHostName(), options.getMongoDBPort()); + if (StringUtils.isEmpty(options.getMongoDBUsername())) { + mongoUrl = + String.format("mongodb://%s:%s", options.getMongoDBHostName(), options.getMongoDBPort()); + } else if (StringUtils.isEmpty(options.getMongoDBPassword())) { + mongoUrl = + String.format( + "mongodb://%s@%s:%s", + options.getMongoDBUsername(), options.getMongoDBHostName(), options.getMongoDBPort()); + } else { + mongoUrl = + String.format( + "mongodb://%s:%s@%s:%s", + options.getMongoDBUsername(), + options.getMongoDBPassword(), + options.getMongoDBHostName(), + options.getMongoDBPort()); + } mongoClient = MongoClients.create(mongoUrl); settings = InfluxDBSettings.builder()