amaliujia commented on a change in pull request #14774:
URL: https://github.com/apache/beam/pull/14774#discussion_r632061480
##########
File path: .test-infra/jenkins/NexmarkBuilder.groovy
##########
@@ -81,16 +87,46 @@ class NexmarkBuilder {
}
- static void suite(context, String title, Runner runner, SDK sdk, Map<String,
Object> options) {
+ static void suite(context, String title, Runner runner, SDK sdk, Map<String,
Object> options, List<String> jobSpecificSwitches, String javaRuntimeVersion) {
+ if (javaRuntimeVersion == JAVA_11_RUNTIME_VERSION) {
+ java11Suite(context, title, runner, sdk, options, jobSpecificSwitches)
+ } else {
+ InfluxDBCredentialsHelper.useCredentials(context)
Review comment:
create a `java8suite` and move this block into taht suite.
##########
File path: .test-infra/jenkins/NexmarkBuilder.groovy
##########
@@ -81,16 +87,46 @@ class NexmarkBuilder {
}
- static void suite(context, String title, Runner runner, SDK sdk, Map<String,
Object> options) {
+ static void suite(context, String title, Runner runner, SDK sdk, Map<String,
Object> options, List<String> jobSpecificSwitches, String javaRuntimeVersion) {
+ if (javaRuntimeVersion == JAVA_11_RUNTIME_VERSION) {
+ java11Suite(context, title, runner, sdk, options, jobSpecificSwitches)
+ } else {
+ InfluxDBCredentialsHelper.useCredentials(context)
+ context.steps {
+ shell("echo \"*** RUN ${title} ***\"")
Review comment:
Run with java 8?
##########
File path: sdks/java/testing/nexmark/build.gradle
##########
@@ -102,6 +109,39 @@ if (shouldProvideSpark) {
}
}
+def getNexmarkArgs = {
+ def nexmarkArgsStr = project.findProperty(nexmarkArgsProperty) ?: ""
+ def nexmarkArgsList = new ArrayList<String>()
+ Collections.addAll(nexmarkArgsList, nexmarkArgsStr.split())
+
+ if (isDataflowRunner) {
+ if (isDataflowRunnerV2) {
+
nexmarkArgsList.add("--experiments=beam_fn_api,use_unified_worker,use_runner_v2,shuffle_mode=appliance")
+ def sdkContainerImage = project.findProperty('sdkContainerImage') ?:
project(":runners:google-cloud-dataflow-java").dockerImageName
+ nexmarkArgsList.add("--sdkContainerImage=${sdkContainerImage}")
Review comment:
Is this line trying to provide images for runner v2? IIRC usually I use
two different flags to specify images for runner v2. Are you referring to other
v2 setup to use this flag?
##########
File path: .test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2.groovy
##########
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as commonJobProperties
+import CommonTestProperties.Runner
+import CommonTestProperties.SDK
+import CommonTestProperties.TriggeringContext
+import NexmarkBuilder as Nexmark
+import NoPhraseTriggeringPostCommitBuilder
+import PhraseTriggeringPostCommitBuilder
+import InfluxDBCredentialsHelper
+
+import static NexmarkDatabaseProperties.nexmarkBigQueryArgs
+import static NexmarkDatabaseProperties.nexmarkInfluxDBArgs
+
+def final JOB_SPECIFIC_OPTIONS = [
+ 'influxTags' :
'{\\\"runnerVersion\\\":\\\"V2\\\",\\\"javaVersion\\\":\\\"8\\\"}',
+ 'exportSummaryToBigQuery' : false,
+ 'region' : 'us-central1',
+ 'suite' : 'STRESS',
Review comment:
Question: there are also other modes. E.g. smoke test. Why choose
`STRESS` (for performance measurement purpose)?
##########
File path: sdks/java/testing/nexmark/build.gradle
##########
@@ -102,6 +109,39 @@ if (shouldProvideSpark) {
}
}
+def getNexmarkArgs = {
+ def nexmarkArgsStr = project.findProperty(nexmarkArgsProperty) ?: ""
+ def nexmarkArgsList = new ArrayList<String>()
+ Collections.addAll(nexmarkArgsList, nexmarkArgsStr.split())
+
+ if (isDataflowRunner) {
+ if (isDataflowRunnerV2) {
+
nexmarkArgsList.add("--experiments=beam_fn_api,use_unified_worker,use_runner_v2,shuffle_mode=appliance")
Review comment:
I would suggest you use `shuffle_mode=service`. Currently we just focus
on runnver v2 + shuffle service.
##########
File path: sdks/java/testing/nexmark/build.gradle
##########
@@ -102,6 +109,39 @@ if (shouldProvideSpark) {
}
}
+def getNexmarkArgs = {
+ def nexmarkArgsStr = project.findProperty(nexmarkArgsProperty) ?: ""
+ def nexmarkArgsList = new ArrayList<String>()
+ Collections.addAll(nexmarkArgsList, nexmarkArgsStr.split())
+
+ if (isDataflowRunner) {
+ if (isDataflowRunnerV2) {
+
nexmarkArgsList.add("--experiments=beam_fn_api,use_unified_worker,use_runner_v2,shuffle_mode=appliance")
+ def sdkContainerImage = project.findProperty('sdkContainerImage') ?:
project(":runners:google-cloud-dataflow-java").dockerImageName
+ nexmarkArgsList.add("--sdkContainerImage=${sdkContainerImage}")
+ // TODO(BEAM-12295) enable all queries once issues with runner V2 is
fixed.
+ if (nexmarkArgsList.contains("--streaming=true")) {
+
nexmarkArgsList.add("--skipQueries=AVERAGE_PRICE_FOR_CATEGORY,AVERAGE_SELLING_PRICE_BY_SELLER,WINNING_BIDS,BOUNDED_SIDE_INPUT_JOIN,SESSION_SIDE_INPUT_JOIN,PORTABILITY_BATCH")
// 4, 6, 9, 13, 14, 15
+ } else {
+
nexmarkArgsList.add("--skipQueries=LOCAL_ITEM_SUGGESTION,AVERAGE_PRICE_FOR_CATEGORY,AVERAGE_SELLING_PRICE_BY_SELLER,HIGHEST_BID,WINNING_BIDS,SESSION_SIDE_INPUT_JOIN,BOUNDED_SIDE_INPUT_JOIN")
// 3, 4, 6, 7, 9, 13, 14, 15
+ }
+ } else {
+ def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath
+ // Provide job with a customizable worker jar.
+ // With legacy worker jar, containerImage is set to empty (i.e. to use
the internal build).
+ // More context and discussions can be found in PR#6694.
+
nexmarkArgsList.add("--dataflowWorkerJar=${dataflowWorkerJar}".toString())
+ nexmarkArgsList.add('--workerHarnessContainerImage=')
Review comment:
This flag seems not set?
##########
File path: .test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2.groovy
##########
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as commonJobProperties
+import CommonTestProperties.Runner
+import CommonTestProperties.SDK
+import CommonTestProperties.TriggeringContext
+import NexmarkBuilder as Nexmark
+import NoPhraseTriggeringPostCommitBuilder
+import PhraseTriggeringPostCommitBuilder
+import InfluxDBCredentialsHelper
+
+import static NexmarkDatabaseProperties.nexmarkBigQueryArgs
+import static NexmarkDatabaseProperties.nexmarkInfluxDBArgs
+
+def final JOB_SPECIFIC_OPTIONS = [
+ 'influxTags' :
'{\\\"runnerVersion\\\":\\\"V2\\\",\\\"javaVersion\\\":\\\"8\\\"}',
+ 'exportSummaryToBigQuery' : false,
+ 'region' : 'us-central1',
+ 'suite' : 'STRESS',
+ 'numWorkers' : 4,
+ 'maxNumWorkers' : 4,
+ 'autoscalingAlgorithm' : 'NONE',
+ 'nexmarkParallel' : 16,
+ 'enforceEncodability' : true,
+ 'enforceImmutability' : true
+]
+
+def final JOB_SPECIFIC_SWITCHES = [
+ '-Pnexmark.runner.version="V2"'
+]
+
+// This job runs the suite of Nexmark tests against the Dataflow runner V2.
+NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Dataflow_V2',
Review comment:
Question: why need a `NoPhraseTriggeringPostCommitBuilder`?
##########
File path: .test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2.groovy
##########
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as commonJobProperties
+import CommonTestProperties.Runner
+import CommonTestProperties.SDK
+import CommonTestProperties.TriggeringContext
+import NexmarkBuilder as Nexmark
+import NoPhraseTriggeringPostCommitBuilder
+import PhraseTriggeringPostCommitBuilder
+import InfluxDBCredentialsHelper
+
+import static NexmarkDatabaseProperties.nexmarkBigQueryArgs
+import static NexmarkDatabaseProperties.nexmarkInfluxDBArgs
+
+def final JOB_SPECIFIC_OPTIONS = [
+ 'influxTags' :
'{\\\"runnerVersion\\\":\\\"V2\\\",\\\"javaVersion\\\":\\\"8\\\"}',
+ 'exportSummaryToBigQuery' : false,
+ 'region' : 'us-central1',
+ 'suite' : 'STRESS',
+ 'numWorkers' : 4,
+ 'maxNumWorkers' : 4,
+ 'autoscalingAlgorithm' : 'NONE',
+ 'nexmarkParallel' : 16,
+ 'enforceEncodability' : true,
+ 'enforceImmutability' : true
+]
+
+def final JOB_SPECIFIC_SWITCHES = [
+ '-Pnexmark.runner.version="V2"'
Review comment:
Ah to confirm, this flag enables runner v2?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
