This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch nightly-refs/heads/master in repository https://gitbox.apache.org/repos/asf/beam.git
from f50c8d503af Deprecate Cloud Debugger and its pipeline options (#25998) add fc7a240eb4e BigQuery Storage Write API wrapper for Python SDK (#25685) add 91bfbac7a90 Update Java and Python base image versions, resolve CI build issue (#25992) add a4f55afa052 Bump tensorflow (#25974) add 08ae6c7acff [Playground] Implement Java & Go multifile examples execution (#24874) add 567bdaa4563 fix: Porting more BigtableOptions (#25955) add 469101dd378 [Playground] "always_run" flag in examples' tag (#25928) add c0f0f2d3c15 Overload HbaseIO with KV<RowKey, RowMutations> (#25864) add 14a77825928 Add explicit language_level=3 arg to Cythonize (#26015) add 6484a47c3d5 Fix TestPipeline.runWithAdditionalOptionArgs ignore additionalArgs (#25937) add 3efd3c38b99 Update pip-licenses in Dockerfile to support Python 3.11 (#26019) add 5bdc4527a47 [#22737] Re-write Go SDK data plane to support timers. (#25982) add 4911a3e150b [Playground] Guard router methods from being executed on runners (#25931) add 7bfbdf2d0f1 Remove dead session code & noisy logging (#22241) (#26029) add 04288fed142 use example id for cloud path (#26027) add 5569085c0b0 [#26030] Add PTransformID to logged messages. (#26031) add 652e30a383b Update container tags (#26022) add c196cd15e99 Update CHANGES.md for HBaseIO.WriteRowMutations (#26032) add 87783393936 Fix small doc typo (#26033) add 9c614557c51 [Go SDK]: Add fileio MatchFiles, MatchAll and ReadMatches transforms (#25809) add 9f582e564fe Add key param support for helper Combiners (#25895) add 52531b73cec Add #24599 to CHANGES.md (#26020) add 915da25323d Update pillow requirement from <8,>=7.1.1 to >=7.1.1,<10 in /sdks/python (#25788) add d9321d8cdb3 Recommend `yield from` for emitting elements from iterables. (#25893) No new revisions were added by this update. Summary of changes: .github/workflows/playground_backend_precommit.yml | 4 +- .../workflows/playground_examples_ci_reusable.yml | 6 +- ...Commit_Python_CrossLanguage_Gcp_Dataflow.groovy | 55 ++ ...stCommit_Python_CrossLanguage_Gcp_Direct.groovy | 55 ++ CHANGES.md | 6 + .../org/apache/beam/gradle/BeamModulePlugin.groovy | 144 +++++ .../backend/playground_api/api/v1/api.pb.go | 588 +++++++++---------- playground/api/v1/api.proto | 1 + playground/backend/cmd/server/controller.go | 56 ++ playground/backend/cmd/server/controller_test.go | 80 ++- playground/backend/containers/java/Dockerfile | 8 +- playground/backend/containers/python/Dockerfile | 4 +- playground/backend/containers/python/entrypoint.sh | 2 +- playground/backend/containers/scio/Dockerfile | 2 +- playground/backend/internal/api/v1/api.pb.go | 588 +++++++++---------- .../internal/code_processing/code_processing.go | 10 +- .../code_processing/code_processing_test.go | 2 +- .../backend/internal/db/datastore/datastore_db.go | 2 +- .../backend/internal/db/dto/precompiled_object.go | 1 + playground/backend/internal/db/entity/example.go | 1 + playground/backend/internal/db/mapper/mapper.go | 2 +- .../db/mapper/precompiled_object_mapper.go | 23 +- .../db/mapper/precompiled_object_mapper_test.go | 2 +- .../internal/environment/environment_service.go | 2 +- .../environment/environment_service_test.go | 18 +- playground/backend/internal/executors/executor.go | 10 +- .../backend/internal/executors/executor_builder.go | 30 +- .../internal/executors/executor_builder_test.go | 2 +- .../backend/internal/executors/executor_test.go | 8 +- playground/backend/internal/fs_tool/fs.go | 6 +- playground/backend/internal/fs_tool/fs_test.go | 4 +- playground/backend/internal/fs_tool/go_fs.go | 4 +- playground/backend/internal/fs_tool/go_fs_test.go | 4 +- playground/backend/internal/fs_tool/java_fs.go | 130 ++++- .../backend/internal/fs_tool/java_fs_test.go | 215 ++++++- .../fs_tool/java_testdata/HasIncorrectMain.java | 28 + .../fs_tool/java_testdata/HasMainTest1.java | 28 + .../fs_tool/java_testdata/HasMainTest2.java | 29 + .../internal/fs_tool/java_testdata/HasNoMain.java | 28 + playground/backend/internal/fs_tool/scio_fs.go | 2 +- .../internal/setup_tools/builder/setup_builder.go | 51 +- .../setup_tools/builder/setup_builder_test.go | 149 ++++- .../setup_tools/life_cycle/life_cycle_setuper.go | 4 +- playground/backend/internal/utils/file_utils.go | 4 + .../lib/src/api/v1/api.pb.dart | 14 + .../lib/src/api/v1/api.pbjson.dart | 3 +- .../lib/src/cache/example_cache.dart | 48 +- .../example_loaders/content_example_loader.dart | 1 + .../controllers/snippet_editing_controller.dart | 1 + .../lib/src/models/example_base.dart | 2 + .../content_example_loading_descriptor.dart | 11 +- .../example_client/grpc_example_client.dart | 1 + .../content_example_loading_descriptor_test.dart | 2 + playground/infrastructure/api/v1/api_pb2.py | 120 ++-- playground/infrastructure/api/v1/api_pb2.pyi | 6 +- playground/infrastructure/datastore_client.py | 10 +- playground/infrastructure/models.py | 1 + sdks/go/pkg/beam/core/metrics/metrics.go | 32 +- sdks/go/pkg/beam/core/runtime/exec/data.go | 16 +- sdks/go/pkg/beam/core/runtime/exec/datasource.go | 182 ++++-- .../pkg/beam/core/runtime/exec/datasource_test.go | 115 ++-- .../go/pkg/beam/core/runtime/exec/dynsplit_test.go | 14 +- sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 514 +++++++++++------ .../pkg/beam/core/runtime/harness/datamgr_test.go | 403 ++++++++++--- sdks/go/pkg/beam/core/runtime/harness/harness.go | 25 +- sdks/go/pkg/beam/core/runtime/harness/logging.go | 92 +-- .../pkg/beam/core/runtime/harness/logging_test.go | 65 ++- sdks/go/pkg/beam/core/runtime/harness/session.go | 232 -------- .../go/pkg/beam/core/runtime/harness/session.proto | 71 --- .../core/runtime/harness/session/session.pb.go | 628 --------------------- sdks/go/pkg/beam/io/fileio/example_test.go | 74 +++ sdks/go/pkg/beam/io/fileio/file.go | 142 +++++ sdks/go/pkg/beam/io/fileio/file_test.go | 254 +++++++++ sdks/go/pkg/beam/io/fileio/gzip.go | 60 ++ sdks/go/pkg/beam/io/fileio/helper_test.go | 88 +++ sdks/go/pkg/beam/io/fileio/match.go | 189 +++++++ sdks/go/pkg/beam/io/fileio/match_test.go | 285 ++++++++++ sdks/go/pkg/beam/io/fileio/read.go | 140 +++++ sdks/go/pkg/beam/io/fileio/read_test.go | 181 ++++++ .../java/org/apache/beam/sdk/schemas/Schema.java | 6 +- .../org/apache/beam/sdk/testing/TestPipeline.java | 23 +- .../org/apache/beam/sdk/schemas/SchemaTest.java | 1 - .../apache/beam/sdk/testing/TestPipelineTest.java | 26 + ...xpansionServiceSchemaTransformProviderTest.java | 14 +- .../expansion-service/build.gradle | 1 - .../io/gcp/bigquery/BeamRowToStorageApiProto.java | 2 +- .../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 11 +- ...ueryStorageWriteApiSchemaTransformProvider.java | 60 +- .../io/gcp/bigtable/BigtableConfigTranslator.java | 9 + .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 8 +- .../sdk/io/gcp/bigtable/BigtableReadOptions.java | 7 + .../io/gcp/bigtable/BigtableServiceFactory.java | 7 +- .../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 24 +- .../sdk/io/gcp/bigtable/BigtableWriteOptions.java | 5 + .../gcp/bigquery/BeamRowToStorageApiProtoTest.java | 2 +- ...StorageWriteApiSchemaTransformProviderTest.java | 155 ++--- .../sdk/io/hbase/HBaseCoderProviderRegistrar.java | 1 + .../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 221 ++++++++ .../beam/sdk/io/hbase/HBaseRowMutationsCoder.java | 150 +++++ .../beam/sdk/io/hbase/HBaseSharedConnection.java | 153 +++++ .../io/hbase/HBaseCoderProviderRegistrarTest.java | 6 + .../sdk/io/hbase/HBaseSharedConnectionTest.java | 100 ++++ .../sdk/io/hbase/HbaseIOWriteRowMutationsTest.java | 214 +++++++ .../beam/sdk/io/hbase/RowMutationsCoderTest.java | 153 +++++ .../apache/beam/sdk/io/hbase/utils/HashUtils.java | 151 +++++ .../beam/sdk/io/hbase/utils/TestConstants.java | 37 ++ .../beam/sdk/io/hbase/utils/TestHBaseUtils.java | 73 +++ .../examples/ml-orchestration/tfx/requirements.txt | 2 +- sdks/python/apache_beam/io/gcp/bigquery.py | 213 ++++++- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 119 ++++ .../apache_beam/io/gcp/bigquery_tools_test.py | 238 ++++++++ .../apache_beam/io/gcp/bigquery_write_it_test.py | 190 +++++++ .../apache_beam/runners/dataflow/internal/names.py | 4 +- sdks/python/apache_beam/transforms/combiners.py | 20 +- .../apache_beam/transforms/combiners_test.py | 14 + sdks/python/apache_beam/transforms/external.py | 30 + .../python/apache_beam/transforms/external_test.py | 43 ++ sdks/python/container/Dockerfile | 2 +- sdks/python/pytest.ini | 1 + sdks/python/setup.py | 9 +- sdks/python/test-suites/dataflow/common.gradle | 24 + sdks/python/test-suites/direct/common.gradle | 25 + sdks/python/test-suites/xlang/build.gradle | 62 ++ settings.gradle.kts | 1 + .../en/documentation/ml/inference-overview.md | 4 +- .../content/en/documentation/programming-guide.md | 7 +- 126 files changed, 6658 insertions(+), 2390 deletions(-) create mode 100644 .test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Dataflow.groovy create mode 100644 .test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Direct.groovy create mode 100644 playground/backend/internal/fs_tool/java_testdata/HasIncorrectMain.java create mode 100644 playground/backend/internal/fs_tool/java_testdata/HasMainTest1.java create mode 100644 playground/backend/internal/fs_tool/java_testdata/HasMainTest2.java create mode 100644 playground/backend/internal/fs_tool/java_testdata/HasNoMain.java delete mode 100644 sdks/go/pkg/beam/core/runtime/harness/session.go delete mode 100644 sdks/go/pkg/beam/core/runtime/harness/session.proto delete mode 100644 sdks/go/pkg/beam/core/runtime/harness/session/session.pb.go create mode 100644 sdks/go/pkg/beam/io/fileio/example_test.go create mode 100644 sdks/go/pkg/beam/io/fileio/file.go create mode 100644 sdks/go/pkg/beam/io/fileio/file_test.go create mode 100644 sdks/go/pkg/beam/io/fileio/gzip.go create mode 100644 sdks/go/pkg/beam/io/fileio/helper_test.go create mode 100644 sdks/go/pkg/beam/io/fileio/match.go create mode 100644 sdks/go/pkg/beam/io/fileio/match_test.go create mode 100644 sdks/go/pkg/beam/io/fileio/read.go create mode 100644 sdks/go/pkg/beam/io/fileio/read_test.go create mode 100644 sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseRowMutationsCoder.java create mode 100644 sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseSharedConnection.java create mode 100644 sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseSharedConnectionTest.java create mode 100644 sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HbaseIOWriteRowMutationsTest.java create mode 100644 sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/RowMutationsCoderTest.java create mode 100644 sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/utils/HashUtils.java create mode 100644 sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/utils/TestConstants.java create mode 100644 sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/utils/TestHBaseUtils.java create mode 100644 sdks/python/test-suites/xlang/build.gradle