This is an automated email from the ASF dual-hosted git repository.
derrickaw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 06e4102173b [yaml] - fix spanner configs and examples (#40587)
06e4102173b is described below
commit 06e4102173b4e4ff72298f3562320e9aec9547ca
Author: Derrick Williams <[email protected]>
AuthorDate: Wed Oct 21 13:31:22 2026 -0400
[yaml] - fix spanner configs and examples (#40587)
---
.../io/gcp/spanner/SpannerReadSchemaTransformProvider.java | 12 ++++++------
.../gcp/spanner/SpannerWriteSchemaTransformProvider.java | 8 ++++----
.../apache_beam/yaml/examples/testing/examples_test.py | 2 +-
.../yaml/examples/transforms/io/spanner_read.yaml | 8 ++++----
.../yaml/examples/transforms/io/spanner_write.yaml | 8 ++++----
.../transforms/ml/enrich_spanner_with_bigquery.yaml | 14 +++++++-------
.../examples/transforms/pipelines/spanner_to_avro.yaml | 6 +++---
.../examples/transforms/pipelines/spanner_to_bigquery.yaml | 6 +++---
8 files changed, 32 insertions(+), 32 deletions(-)
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadSchemaTransformProvider.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadSchemaTransformProvider.java
index 73f4c2dfe30..f9b63330634 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadSchemaTransformProvider.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerReadSchemaTransformProvider.java
@@ -95,8 +95,8 @@ public class SpannerReadSchemaTransformProvider
+ "\n"
+ " - type: ReadFromSpanner\n"
+ " config:\n"
- + " instance_id: 'my-instance-id'\n"
- + " database_id: 'my-database'\n"
+ + " instance: 'my-instance-id'\n"
+ + " database: 'my-database'\n"
+ " query: 'SELECT * FROM table'\n"
+ "\n"
+ "It is also possible to read a table by specifying a table name and
a list of columns. For "
@@ -104,8 +104,8 @@ public class SpannerReadSchemaTransformProvider
+ "\n"
+ " - type: ReadFromSpanner\n"
+ " config:\n"
- + " instance_id: 'my-instance-id'\n"
- + " database_id: 'my-database'\n"
+ + " instance: 'my-instance-id'\n"
+ + " database: 'my-database'\n"
+ " table: 'my-table'\n"
+ " columns: ['col1', 'col2']\n"
+ "\n"
@@ -114,8 +114,8 @@ public class SpannerReadSchemaTransformProvider
+ "\n"
+ " - type: ReadFromSpanner\n"
+ " config:\n"
- + " instance_id: 'my-instance-id'\n"
- + " database_id: 'my-database'\n"
+ + " instance: 'my-instance-id'\n"
+ + " database: 'my-database'\n"
+ " table: 'my-table'\n"
+ " index: 'my-index'\n"
+ " columns: ['col1', 'col2']\n"
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java
index eee13b0a932..63fc7574f41 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteSchemaTransformProvider.java
@@ -84,11 +84,11 @@ public class SpannerWriteSchemaTransformProvider
+ "\n"
+ "Example configuration for performing a write to a single table:
::\n"
+ "\n"
- + " - type: ReadFromSpanner\n"
+ + " - type: WriteToSpanner\n"
+ " config:\n"
- + " project_id: 'my-project-id'\n"
- + " instance_id: 'my-instance-id'\n"
- + " database_id: 'my-database'\n"
+ + " project: 'my-project-id'\n"
+ + " instance: 'my-instance-id'\n"
+ + " database: 'my-database'\n"
+ " table: 'my-table'\n"
+ "\n"
+ "Note: See <a href=\""
diff --git a/sdks/python/apache_beam/yaml/examples/testing/examples_test.py
b/sdks/python/apache_beam/yaml/examples/testing/examples_test.py
index 95dd9ed86ef..7c81966a2db 100644
--- a/sdks/python/apache_beam/yaml/examples/testing/examples_test.py
+++ b/sdks/python/apache_beam/yaml/examples/testing/examples_test.py
@@ -866,7 +866,7 @@ def _spanner_io_read_test_preprocessor(
for transform in pipeline.get('transforms', []):
if transform.get('type', '').startswith('ReadFromSpanner'):
config = transform['config']
- instance, database = config['instance_id'], config['database_id']
+ instance, database = config['instance'], config['database']
if (table := config.get('table')) is None:
table = config.get('query', '').split('FROM')[-1].strip()
transform['type'] = 'Create'
diff --git
a/sdks/python/apache_beam/yaml/examples/transforms/io/spanner_read.yaml
b/sdks/python/apache_beam/yaml/examples/transforms/io/spanner_read.yaml
index 26f68b68d93..9777d85df36 100644
--- a/sdks/python/apache_beam/yaml/examples/transforms/io/spanner_read.yaml
+++ b/sdks/python/apache_beam/yaml/examples/transforms/io/spanner_read.yaml
@@ -20,14 +20,14 @@ pipeline:
# Reading data from a Spanner database. The table used here has the
following columns:
# shipment_id (String), customer_id (String), shipment_date (String),
shipment_cost (Float64), customer_name (String), customer_email (String)
- # ReadFromSpanner transform is called using project_id, instance_id,
database_id and a query
+ # ReadFromSpanner transform is called using project, instance, database
and a query
# A table with a list of columns can also be specified instead of a query
- type: ReadFromSpanner
name: ReadShipments
config:
- project_id: 'apache-beam-testing'
- instance_id: 'shipment-test'
- database_id: 'shipment'
+ project: 'apache-beam-testing'
+ instance: 'shipment-test'
+ database: 'shipment'
query: 'SELECT * FROM shipments'
# Filtering the data based on a specific condition
diff --git
a/sdks/python/apache_beam/yaml/examples/transforms/io/spanner_write.yaml
b/sdks/python/apache_beam/yaml/examples/transforms/io/spanner_write.yaml
index 1667fcfcc16..4e5a941118d 100644
--- a/sdks/python/apache_beam/yaml/examples/transforms/io/spanner_write.yaml
+++ b/sdks/python/apache_beam/yaml/examples/transforms/io/spanner_write.yaml
@@ -39,10 +39,10 @@ pipeline:
name: WriteSpanner
input: CreateRows
config:
- project_id: 'apache-beam-testing'
- instance_id: 'shipment-test'
- database_id: 'shipment'
- table_id: 'shipments'
+ project: 'apache-beam-testing'
+ instance: 'shipment-test'
+ database: 'shipment'
+ table: 'shipments'
error_handling:
output: my_error_output
diff --git
a/sdks/python/apache_beam/yaml/examples/transforms/ml/enrich_spanner_with_bigquery.yaml
b/sdks/python/apache_beam/yaml/examples/transforms/ml/enrich_spanner_with_bigquery.yaml
index e9c140a7965..f1e161b1287 100644
---
a/sdks/python/apache_beam/yaml/examples/transforms/ml/enrich_spanner_with_bigquery.yaml
+++
b/sdks/python/apache_beam/yaml/examples/transforms/ml/enrich_spanner_with_bigquery.yaml
@@ -22,9 +22,9 @@ pipeline:
- type: ReadFromSpanner
name: ReadOrders
config:
- project_id: 'apache-beam-testing'
- instance_id: 'orders-test'
- database_id: 'order-database'
+ project: 'apache-beam-testing'
+ instance: 'orders-test'
+ database: 'order-database'
query: 'SELECT customer_id, product_id, order_date, order_amount FROM
orders'
# Step 2: Enrich order details with customers details from BigQuery
@@ -82,10 +82,10 @@ pipeline:
name: WriteProcessedOrders
input: FilterHighValueOrders
config:
- project_id: '$PROJECT'
- instance_id: '$INSTANCE'
- database_id: '$DATABASE'
- table_id: '$TABLE'
+ project: '$PROJECT'
+ instance: '$INSTANCE'
+ database: '$DATABASE'
+ table: '$TABLE'
error_handling:
output: my_error_output
diff --git
a/sdks/python/apache_beam/yaml/examples/transforms/pipelines/spanner_to_avro.yaml
b/sdks/python/apache_beam/yaml/examples/transforms/pipelines/spanner_to_avro.yaml
index eade572fa45..71415154445 100644
---
a/sdks/python/apache_beam/yaml/examples/transforms/pipelines/spanner_to_avro.yaml
+++
b/sdks/python/apache_beam/yaml/examples/transforms/pipelines/spanner_to_avro.yaml
@@ -26,9 +26,9 @@ pipeline:
- type: ReadFromSpanner
name: ReadShipments
config:
- project_id: 'apache-beam-testing'
- instance_id: 'shipment-test'
- database_id: 'shipment'
+ project: 'apache-beam-testing'
+ instance: 'shipment-test'
+ database: 'shipment'
query: 'SELECT * FROM shipments'
# Step 2: Write successful records out to GCS avro files
- type: WriteToAvro
diff --git
a/sdks/python/apache_beam/yaml/examples/transforms/pipelines/spanner_to_bigquery.yaml
b/sdks/python/apache_beam/yaml/examples/transforms/pipelines/spanner_to_bigquery.yaml
index 7da5058c3ad..d3497b783bf 100644
---
a/sdks/python/apache_beam/yaml/examples/transforms/pipelines/spanner_to_bigquery.yaml
+++
b/sdks/python/apache_beam/yaml/examples/transforms/pipelines/spanner_to_bigquery.yaml
@@ -26,9 +26,9 @@ pipeline:
- type: ReadFromSpanner
name: ReadShipments
config:
- project_id: 'apache-beam-testing'
- instance_id: 'shipment-test'
- database_id: 'shipment'
+ project: 'apache-beam-testing'
+ instance: 'shipment-test'
+ database: 'shipment'
query: 'SELECT * FROM shipments'
# Step 2: Write records out to BigQuery
- type: WriteToBigQuery
