This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 8e49c5a [BEAM-8472] Get default GCP region from gcloud (Java) new df0e7e8 Merge pull request #9974 from ibzib/java-default-region 8e49c5a is described below commit 8e49c5a50cac90bbc77f05d24e99b53d87f5191f Author: Kyle Weaver <kcwea...@google.com> AuthorDate: Thu Oct 31 17:27:22 2019 -0700 [BEAM-8472] Get default GCP region from gcloud (Java) --- .../beam/runners/dataflow/DataflowRunner.java | 8 --- .../dataflow/options/DataflowPipelineOptions.java | 57 +++++++++++++++++++++- .../options/DataflowPipelineOptionsTest.java | 6 +++ 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java index 54b93c7..3206870 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java @@ -247,14 +247,6 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> { "Missing required values: " + Joiner.on(',').join(missing)); } - if (dataflowOptions.getRegion() == null) { - dataflowOptions.setRegion("us-central1"); - LOG.warn( - "--region not set; will default to us-central1. Future releases of Beam will " - + "require the user to set the region explicitly. " - + "https://cloud.google.com/compute/docs/regions-zones/regions-zones"); - } - validateWorkerSettings(PipelineOptionsValidator.validate(GcpOptions.class, options)); PathValidator validator = dataflowOptions.getPathValidator(); diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java index c035839..4353bbb 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java @@ -17,7 +17,13 @@ */ package org.apache.beam.runners.dataflow.options; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; @@ -118,8 +124,6 @@ public interface DataflowPipelineOptions * The Google Compute Engine <a * href="https://cloud.google.com/compute/docs/regions-zones/regions-zones">region</a> for * creating Dataflow jobs. - * - * <p>NOTE: The Cloud Dataflow now also supports the region flag. */ @Hidden @Experimental @@ -128,6 +132,7 @@ public interface DataflowPipelineOptions + "https://cloud.google.com/compute/docs/regions-zones/regions-zones for a list of valid " + "options. Currently defaults to us-central1, but future releases of Beam will " + "require the user to set the region explicitly.") + @Default.InstanceFactory(DefaultGcpRegionFactory.class) String getRegion(); void setRegion(String region); @@ -201,4 +206,52 @@ public interface DataflowPipelineOptions .toString(); } } + + /** + * Factory for a default value for Google Cloud region according to + * https://cloud.google.com/compute/docs/gcloud-compute/#default-properties. If no other default + * can be found, returns "us-central1". + */ + class DefaultGcpRegionFactory implements DefaultValueFactory<String> { + private static final Logger LOG = LoggerFactory.getLogger(DefaultGcpRegionFactory.class); + + @Override + public String create(PipelineOptions options) { + String environmentRegion = System.getenv("CLOUDSDK_COMPUTE_REGION"); + if (environmentRegion != null && !environmentRegion.isEmpty()) { + LOG.info("Using default GCP region {} from $CLOUDSDK_COMPUTE_REGION", environmentRegion); + return environmentRegion; + } + try { + ProcessBuilder pb = + new ProcessBuilder(Arrays.asList("gcloud", "config", "get-value", "compute/region")); + Process process = pb.start(); + try (BufferedReader reader = + new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)); + BufferedReader errorReader = + new BufferedReader( + new InputStreamReader(process.getErrorStream(), StandardCharsets.UTF_8))) { + if (process.waitFor(2, TimeUnit.SECONDS) && process.exitValue() == 0) { + String gcloudRegion = reader.lines().collect(Collectors.joining()); + if (!gcloudRegion.isEmpty()) { + LOG.info("Using default GCP region {} from gcloud CLI", gcloudRegion); + return gcloudRegion; + } + } else { + String stderr = errorReader.lines().collect(Collectors.joining("\n")); + LOG.debug("gcloud exited with exit value {}. Stderr:\n{}", process.exitValue(), stderr); + } + } + } catch (Exception e) { + // Ignore. + LOG.debug("Unable to get gcloud compute region", e); + } + LOG.warn( + "Region will default to us-central1. Future releases of Beam will " + + "require the user to set the region explicitly. " + + "https://cloud.google.com/compute/docs/regions-zones/regions-zones"); + return "us-central1"; + } + } } diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java index 754f061..1bf5cb3 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java @@ -199,4 +199,10 @@ public class DataflowPipelineOptionsTest { thrown.expectMessage("Error constructing default value for stagingLocation"); options.getStagingLocation(); } + + @Test + public void testDefaultGcpRegion() { + DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); + assertEquals("us-central1", options.getRegion()); + } }