This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch nightly-refs/heads/master in repository https://gitbox.apache.org/repos/asf/beam.git
from 2b309e87e08 Bump worker image dependency (#27253) add d0806a6807e [#22737] Add line about timer support for Go (#27263) add aba3225d0f7 Bump actions/checkout from 2 to 3 (#27259) add 3844587819c Add Tour of Beam Page (#27244) add 0db241d457b Bump transformers (#27106) add 2a06a270ac5 Bump google.golang.org/protobuf from 1.30.0 to 1.31.0 in /sdks (#27262) add 4ce16432090 [prism] Fix linting issue for flag var (#27248) add c1a5ce7bf0a Add notebook for image processing using beam (#27034) add 8f21e9e9990 Enable remote caching for Beam builds (#27015) add ee9d37daa97 Remove some null checks for optional SplunkIO parameters (#27177) add 8695d704a59 don't error-log expected offset mismatch errors (#27251) add 61651c3394c Update Release Guide DockerHub beammaintainers team to beam (#27264) add 270e4fe2e2e Add error metrics and DLQ support to GCS/File write schema transforms (#27039) add 2f555c51f18 Disable FirestoreIT tests on Dataflow (#27267) add 664f355ee9c Fix perf tool tests (#27270) add 084585abc81 Bump google.golang.org/api from 0.128.0 to 0.129.0 in /sdks (#27265) add eac3d63738a Use a more mnemonic variable name (#27217) add bc14623d02f Bump cloud.google.com/go/bigquery from 1.51.2 to 1.52.0 in /sdks (#27266) add 0c55d382988 Start expansion service containers before the controller container add 4ff952afce7 Merge pull request #27273: Start expansion service containers before the controller container add a2ebc8a95f1 Cleanup for GA launch of ReadChangeStream (#27249) add 5f1eae62293 Decrease run log failures severity (#27230) add 6cd15e7bed1 BigTable write SchemaTransform and Python wrapper (#27146) No new revisions were added by this update. Summary of changes: .github/workflows/build_runner_image.yml | 2 +- .github/workflows/run_perf_alert_tool.yml | 6 + .test-infra/jenkins/CommonJobProperties.groovy | 1 + ...Commit_Python_CrossLanguage_Gcp_Dataflow.groovy | 2 +- CHANGES.md | 2 + .../beam-ml/image_processing_tensorflow.ipynb | 951 +++++++++++++++++++++ .../internal/code_processing/code_processing.go | 5 +- .../backend/internal/errors/lifecycle_error.go | 13 + runners/google-cloud-dataflow-java/build.gradle | 4 + sdks/go.mod | 12 +- sdks/go.sum | 26 +- sdks/go/cmd/prism/prism.go | 4 +- .../AvroWriteSchemaTransformFormatProvider.java | 45 +- .../CsvWriteSchemaTransformFormatProvider.java | 16 +- .../FileWriteSchemaTransformFormatProvider.java | 6 +- .../FileWriteSchemaTransformFormatProviders.java | 55 ++ .../FileWriteSchemaTransformProvider.java | 18 +- .../JsonWriteSchemaTransformFormatProvider.java | 41 +- .../ParquetWriteSchemaTransformFormatProvider.java | 45 +- .../XmlWriteSchemaTransformFormatProvider.java | 37 +- ...AvroWriteSchemaTransformFormatProviderTest.java | 79 +- .../CsvWriteSchemaTransformFormatProviderTest.java | 9 +- ...FileWriteSchemaTransformFormatProviderTest.java | 4 +- ...ileWriteSchemaTransformFormatProvidersTest.java | 86 ++ ...JsonWriteSchemaTransformFormatProviderTest.java | 60 ++ ...quetWriteSchemaTransformFormatProviderTest.java | 75 ++ .../XmlWriteSchemaTransformFormatProviderTest.java | 43 + .../bigquery/StorageApiWritesShardedRecords.java | 33 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 133 ++- .../BigtableWriteSchemaTransformProvider.java | 253 ++++++ .../changestreams/action/ChangeStreamAction.java | 41 +- .../action/ReadChangeStreamPartitionAction.java | 31 +- .../dao/BigtableChangeStreamAccessor.java | 44 +- .../changestreams/dao/ChangeStreamDao.java | 14 +- .../gcp/bigtable/changestreams/dao/DaoFactory.java | 15 +- .../changestreams/dao/MetadataTableAdminDao.java | 5 + .../changestreams/dao/MetadataTableDao.java | 47 +- .../changestreams/dofn/InitializeDoFn.java | 35 +- .../dofn/ReadChangeStreamPartitionDoFn.java | 11 +- .../BigtableWriteSchemaTransformProviderIT.java | 413 +++++++++ .../action/ChangeStreamActionTest.java | 12 +- .../ReadChangeStreamPartitionActionTest.java | 33 +- .../changestreams/dao/MetadataTableDaoTest.java | 31 + .../changestreams/dofn/InitializeDoFnTest.java | 71 +- .../dofn/ReadChangeStreamPartitionDoFnTest.java | 5 +- .../sdk/io/gcp/firestore/it/BaseFirestoreIT.java | 3 - .../org/apache/beam/sdk/io/splunk/SplunkIO.java | 24 - .../docker-compose/docker-compose.yml | 3 + .../inference/runinference_metrics/setup.py | 2 +- .../apache_beam/io/gcp/bigquery_schema_tools.py | 8 +- sdks/python/apache_beam/io/gcp/bigtableio.py | 135 ++- .../apache_beam/io/gcp/bigtableio_it_test.py | 377 ++++++++ sdks/python/apache_beam/io/gcp/bigtableio_test.py | 247 +++--- .../apache_beam/testing/analyzers/perf_analysis.py | 6 +- .../testing/analyzers/perf_analysis_test.py | 14 +- settings.gradle.kts | 11 +- .../site/content/en/contribute/release-guide.md | 12 +- .../tour-of-beam.md} | 10 +- 58 files changed, 3195 insertions(+), 531 deletions(-) create mode 100644 examples/notebooks/beam-ml/image_processing_tensorflow.ipynb create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProvider.java create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProviderIT.java create mode 100644 sdks/python/apache_beam/io/gcp/bigtableio_it_test.py copy website/www/site/content/en/{documentation/transforms/python/aggregation/approximatequantiles.md => get-started/tour-of-beam.md} (66%)