This is an automated email from the ASF dual-hosted git repository. pabloem pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 2354edd Update V1Beta3 API and add DebugOptions new b43952c Merge pull request #14380 from [BEAM-10994] Java: Update V1Beta3 API and add DebugOptions 2354edd is described below commit 2354eddcaf8659b521789b25878780f89c92956f Author: Sam R <rohde.sam...@gmail.com> AuthorDate: Tue Mar 30 21:41:07 2021 +0000 Update V1Beta3 API and add DebugOptions --- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +- .../dataflow/DataflowPipelineTranslator.java | 6 ++++++ .../dataflow/DataflowPipelineTranslatorTest.java | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 5acc301..4aded65 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -512,7 +512,7 @@ class BeamModulePlugin implements Plugin<Project> { google_api_services_bigquery : "com.google.apis:google-api-services-bigquery:v2-rev20210219-$google_clients_version", google_api_services_clouddebugger : "com.google.apis:google-api-services-clouddebugger:v2-rev20200807-$google_clients_version", google_api_services_cloudresourcemanager : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210222-$google_clients_version", - google_api_services_dataflow : "com.google.apis:google-api-services-dataflow:v1b3-rev20210217-$google_clients_version", + google_api_services_dataflow : "com.google.apis:google-api-services-dataflow:v1b3-rev20210301-$google_clients_version", google_api_services_healthcare : "com.google.apis:google-api-services-healthcare:v1beta1-rev20210217-$google_clients_version", google_api_services_pubsub : "com.google.apis:google-api-services-pubsub:v1-rev20210208-$google_clients_version", google_api_services_storage : "com.google.apis:google-api-services-storage:v1-rev20210127-$google_clients_version", diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java index 0fd8d80..23147e7 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java @@ -36,6 +36,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.api.services.dataflow.model.AutoscalingSettings; import com.google.api.services.dataflow.model.DataflowPackage; +import com.google.api.services.dataflow.model.DebugOptions; import com.google.api.services.dataflow.model.Disk; import com.google.api.services.dataflow.model.Environment; import com.google.api.services.dataflow.model.Job; @@ -435,6 +436,11 @@ public class DataflowPipelineTranslator { if (options.getDataflowKmsKey() != null) { environment.setServiceKmsKeyName(options.getDataflowKmsKey()); } + if (options.isHotKeyLoggingEnabled()) { + DebugOptions debugOptions = new DebugOptions(); + debugOptions.setEnableHotKeyLogging(true); + environment.setDebugOptions(debugOptions); + } pipeline.traverseTopologically(this); return job; diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java index 8015ab3..ca306cf 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java @@ -1317,6 +1317,28 @@ public class DataflowPipelineTranslatorTest implements Serializable { assertEquals(serviceOptions, job.getEnvironment().getServiceOptions()); } + @Test + public void testHotKeyLoggingEnabledOption() throws IOException { + DataflowPipelineOptions options = buildPipelineOptions(); + options.setHotKeyLoggingEnabled(true); + + Pipeline p = buildPipeline(options); + p.traverseTopologically(new RecordingPipelineVisitor()); + SdkComponents sdkComponents = createSdkComponents(options); + RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(p, sdkComponents, true); + Job job = + DataflowPipelineTranslator.fromOptions(options) + .translate( + p, + pipelineProto, + sdkComponents, + DataflowRunner.fromOptions(options), + Collections.emptyList()) + .getJob(); + + assertTrue(job.getEnvironment().getDebugOptions().getEnableHotKeyLogging()); + } + private static void assertAllStepOutputsHaveUniqueIds(Job job) throws Exception { List<String> outputIds = new ArrayList<>(); for (Step step : job.getSteps()) {