Repository: beam Updated Branches: refs/heads/master 99b5b0563 -> 52cea71ed
GCP IO ITs now all use --project option Up until now, some IO ITs used --projectId and others used --project This mixing meant that running all the tests in one test run was impossible. Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0acfe704 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0acfe704 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0acfe704 Branch: refs/heads/master Commit: 0acfe70439b50184a69601ca4bb8cff9780fa4ef Parents: 99b5b05 Author: Stephen Sisk <s...@google.com> Authored: Wed Jun 28 15:34:45 2017 -0700 Committer: Stephen Sisk <s...@google.com> Committed: Thu Jun 29 13:24:45 2017 -0700 ---------------------------------------------------------------------- .../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 5 ++++- .../sdk/io/gcp/bigtable/BigtableTestOptions.java | 5 ----- .../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +++- .../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 16 ++++++++-------- .../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 15 +++++++-------- 5 files changed, 22 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/0acfe704/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadIT.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadIT.java index a064bd6..e47fd0f 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadIT.java @@ -20,6 +20,7 @@ package org.apache.beam.sdk.io.gcp.bigtable; import com.google.bigtable.v2.Row; import com.google.cloud.bigtable.config.BigtableOptions; import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; @@ -41,8 +42,10 @@ public class BigtableReadIT { BigtableTestOptions options = TestPipeline.testingPipelineOptions() .as(BigtableTestOptions.class); + String project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); + BigtableOptions.Builder bigtableOptionsBuilder = new BigtableOptions.Builder() - .setProjectId(options.getProjectId()) + .setProjectId(project) .setInstanceId(options.getInstanceId()); final String tableId = "BigtableReadTest"; http://git-wip-us.apache.org/repos/asf/beam/blob/0acfe704/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestOptions.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestOptions.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestOptions.java index 0ab7576..03cb697 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestOptions.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableTestOptions.java @@ -25,11 +25,6 @@ import org.apache.beam.sdk.testing.TestPipelineOptions; * Properties needed when using Bigtable with the Beam SDK. */ public interface BigtableTestOptions extends TestPipelineOptions { - @Description("Project ID for Bigtable") - @Default.String("apache-beam-testing") - String getProjectId(); - void setProjectId(String value); - @Description("Instance ID for Bigtable") @Default.String("beam-test") String getInstanceId(); http://git-wip-us.apache.org/repos/asf/beam/blob/0acfe704/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java index 1d168f1..72ba836 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteIT.java @@ -73,15 +73,17 @@ public class BigtableWriteIT implements Serializable { private static BigtableTableAdminClient tableAdminClient; private final String tableId = String.format("BigtableWriteIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()); + private String project; @Before public void setup() throws Exception { PipelineOptionsFactory.register(BigtableTestOptions.class); options = TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class); + project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); bigtableOptions = new Builder() - .setProjectId(options.getProjectId()) + .setProjectId(project) .setInstanceId(options.getInstanceId()) .setUserAgent("apache-beam-test") .build(); http://git-wip-us.apache.org/repos/asf/beam/blob/0acfe704/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadIT.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadIT.java index ca43b40..32183f9 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadIT.java @@ -31,6 +31,8 @@ import com.google.spanner.admin.database.v1.CreateDatabaseMetadata; import java.util.ArrayList; import java.util.Collections; import java.util.List; + +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.PipelineOptionsFactory; @@ -59,11 +61,6 @@ public class SpannerReadIT { /** Pipeline options for this test. */ public interface SpannerTestPipelineOptions extends TestPipelineOptions { - @Description("Project ID for Spanner") - @Default.String("apache-beam-testing") - String getProjectId(); - void setProjectId(String value); - @Description("Instance ID to write to in Spanner") @Default.String("beam-test") String getInstanceId(); @@ -84,13 +81,16 @@ public class SpannerReadIT { private DatabaseAdminClient databaseAdminClient; private SpannerTestPipelineOptions options; private String databaseName; + private String project; @Before public void setUp() throws Exception { PipelineOptionsFactory.register(SpannerTestPipelineOptions.class); options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class); - spanner = SpannerOptions.newBuilder().setProjectId(options.getProjectId()).build().getService(); + project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); + + spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService(); databaseName = generateDatabaseName(); @@ -118,7 +118,7 @@ public class SpannerReadIT { DatabaseClient databaseClient = spanner.getDatabaseClient( DatabaseId.of( - options.getProjectId(), options.getInstanceId(), databaseName)); + project, options.getInstanceId(), databaseName)); List<Mutation> mutations = new ArrayList<>(); for (int i = 0; i < 5L; i++) { @@ -134,7 +134,7 @@ public class SpannerReadIT { databaseClient.writeAtLeastOnce(mutations); SpannerConfig spannerConfig = SpannerConfig.create() - .withProjectId(options.getProjectId()) + .withProjectId(project) .withInstanceId(options.getInstanceId()) .withDatabaseId(databaseName); http://git-wip-us.apache.org/repos/asf/beam/blob/0acfe704/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java index 613756c..78a360f 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java @@ -33,6 +33,7 @@ import com.google.cloud.spanner.SpannerOptions; import com.google.cloud.spanner.Statement; import com.google.spanner.admin.database.v1.CreateDatabaseMetadata; import java.util.Collections; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; @@ -59,11 +60,6 @@ public class SpannerWriteIT { /** Pipeline options for this test. */ public interface SpannerTestPipelineOptions extends TestPipelineOptions { - @Description("Project ID for Spanner") - @Default.String("apache-beam-testing") - String getProjectId(); - void setProjectId(String value); - @Description("Instance ID to write to in Spanner") @Default.String("beam-test") String getInstanceId(); @@ -84,13 +80,16 @@ public class SpannerWriteIT { private DatabaseAdminClient databaseAdminClient; private SpannerTestPipelineOptions options; private String databaseName; + private String project; @Before public void setUp() throws Exception { PipelineOptionsFactory.register(SpannerTestPipelineOptions.class); options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class); - spanner = SpannerOptions.newBuilder().setProjectId(options.getProjectId()).build().getService(); + project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); + + spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService(); databaseName = generateDatabaseName(); @@ -128,7 +127,7 @@ public class SpannerWriteIT { .apply(ParDo.of(new GenerateMutations(options.getTable()))) .apply( SpannerIO.write() - .withProjectId(options.getProjectId()) + .withProjectId(project) .withInstanceId(options.getInstanceId()) .withDatabaseId(databaseName)); @@ -136,7 +135,7 @@ public class SpannerWriteIT { DatabaseClient databaseClient = spanner.getDatabaseClient( DatabaseId.of( - options.getProjectId(), options.getInstanceId(), databaseName)); + project, options.getInstanceId(), databaseName)); ResultSet resultSet = databaseClient