This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch nightly-refs/heads/master in repository https://gitbox.apache.org/repos/asf/beam.git
from 94601a56d67 fix reference to StorageWriteToBigQuery (#26487) add 4a09ea94727 [Tour of Beam] [Frontend] Markdown selection fix (#26496) add 885f20c43dd Use vendor guava for sdks/java/extensions/sql (#26463) add a13749fd529 [Python] Allow users to pass service name for profiler (#26220) add 68ed7ce48b2 [Playground] Move modify saved snippets functionality to to cloudfunctions (#26026) add 7a93e217f50 [Tour of Beam] [Frontend] Cache examples when switching tabs (#26514) add e868c8da79a Update mass_comment.py when we cut release branch (#26506) add 243c795a843 Add timeout parameter to with_exception_handling. (#26526) add 03ecb158f9a Add confirmation message, remove bad rc_number (#26530) add 5b44d3637c8 [Playground] derf Stop tickers to avoid leaks. (#26507) No new revisions were added by this update. Summary of changes: .github/workflows/cut_release_branch.yml | 43 +++++- .gitignore | 1 + CHANGES.md | 1 + build.gradle.kts | 3 + learning/tour-of-beam/backend/docker-compose.yml | 1 + .../pages/tour/widgets/markdown/code_builder.dart | 2 +- .../pages/tour/widgets/markdown/tob_markdown.dart | 33 ++-- learning/tour-of-beam/frontend/pubspec.lock | 8 +- learning/tour-of-beam/frontend/pubspec.yaml | 7 +- playground/backend/README.md | 1 - playground/backend/build.gradle.kts | 4 +- .../backend/cmd/migration_tool/migration_tool.go | 56 +++++++ .../remove_unused_snippets.go | 5 +- playground/backend/cmd/server/controller_test.go | 21 +-- playground/backend/cmd/server/server.go | 37 ++--- playground/backend/containers/router/Dockerfile | 5 + .../containers/router/docker-compose.local.yml | 1 + playground/backend/containers/router/entrypoint.sh | 15 ++ playground/backend/functions.go | 118 +++++++++++++++ playground/backend/go.mod | 11 +- playground/backend/go.sum | 30 +++- .../backend/internal/cache/local/local_cache.go | 1 + .../internal/code_processing/code_processing.go | 6 +- .../backend/internal/db/datastore/datastore_db.go | 136 ++++++++++------- .../internal/db/datastore/datastore_db_test.go | 51 +------ .../internal/db/datastore/emulator_wrapper.go | 4 +- .../migration_base.go} | 28 ++-- .../backend/internal/db/datastore/migration_db.go | 157 +++++++++++++++++++ playground/backend/internal/db/db.go | 15 +- playground/backend/internal/db/entity/schema.go | 3 +- .../internal/db/mapper/datastore_mapper_test.go | 6 +- .../db/schema/migration/migrations_test.go | 121 --------------- .../db/schema/{migration => }/migration_v001.go | 36 +++-- .../db/{entity/schema.go => schema/migrations.go} | 9 +- playground/backend/internal/db/schema/version.go | 62 -------- playground/backend/internal/emulators/kafka.go | 7 +- .../backend/internal/environment/application.go | 67 +++++++-- .../internal/environment/environment_service.go | 112 ++++++++------ .../environment/environment_service_test.go | 160 ++++++++++++++++++-- .../backend/internal/environment/property.go | 2 - .../backend/internal/environment/property_test.go | 3 +- .../external_functions_component.go | 127 ++++++++++++++++ playground/backend/internal/tasks/task.go | 6 +- .../backend/internal/utils/datastore_utils.go | 7 +- playground/backend/playground_functions/Dockerfile | 38 +++++ .../schema.go => playground_functions/cmd/main.go} | 21 ++- .../func_enviornment.go} | 23 ++- .../middleware.go} | 17 ++- playground/backend/properties.yaml | 2 - playground/docker-compose.local.yaml | 43 ++++++ .../assets/translations/en.yaml | 2 +- .../lib/src/cache/example_cache.dart | 85 ++++++++--- .../example_loaders/standard_example_loader.dart | 6 +- .../lib/src/controllers/playground_controller.dart | 2 +- .../lib/src/models/example_base.g.dart | 2 + .../playground_components/lib/src/theme/theme.dart | 2 +- .../frontend/playground_components/pubspec.yaml | 7 +- .../test/src/common/example_cache.mocks.dart | 132 ++++++++-------- .../src/common/example_repository_mock.mocks.dart | 128 +++++++--------- .../examples_loader_test.mocks.dart | 140 ++++++++--------- .../http_example_loader_test.mocks.dart | 132 ++++++++-------- .../playground_controller_test.mocks.dart | 166 ++++++++++---------- playground/frontend/pubspec.lock | 8 +- .../states/example_selector_state_test.mocks.dart | 2 +- playground/index.yaml | 1 - .../templates/deployment-router.yml | 6 + playground/terraform/README.md | 8 +- playground/terraform/build.gradle.kts | 79 +++++++++- .../infrastructure/api_enable/variables.tf | 2 +- .../{ip_address => archive_file}/main.tf | 10 +- .../infrastructure/cloudfunctions/main.tf | 65 ++++++++ .../variables.tf => cloudfunctions/output.tf} | 13 +- .../{api_enable => cloudfunctions}/variables.tf | 24 ++- .../variables.tf => gke_bucket/main.tf} | 18 ++- .../{ip_address/main.tf => gke_bucket/output.tf} | 4 +- .../{ip_address => gke_bucket}/variables.tf | 9 +- .../terraform/infrastructure/ip_address/main.tf | 2 +- .../infrastructure/ip_address/variables.tf | 2 +- playground/terraform/infrastructure/main.tf | 30 +++- playground/terraform/infrastructure/output.tf | 14 +- playground/terraform/infrastructure/setup/iam.tf | 18 ++- .../terraform/infrastructure/setup/output.tf | 6 +- playground/terraform/infrastructure/variables.tf | 6 +- playground/terraform/main.tf | 6 +- playground/terraform/output.tf | 14 +- playground/terraform/variables.tf | 7 +- .../src/main/scripts/build_release_candidate.sh | 3 +- release/src/main/scripts/jenkins_jobs.txt | 167 +++++++++++++++++++++ release/src/main/scripts/mass_comment.py | 165 ++------------------ release/src/main/scripts/run_rc_validation.sh | 52 ++++--- sdks/java/extensions/sql/datacatalog/build.gradle | 1 - .../sql/example/BeamSqlDataCatalogExample.java | 2 +- .../datacatalog/DataCatalogTableProvider.java | 2 +- .../sql/meta/provider/datacatalog/SchemaUtils.java | 7 +- .../beam/sdk/extensions/sql/SqlTransform.java | 4 +- .../sql/impl/BeamCalciteSchemaFactory.java | 2 +- .../sdk/extensions/sql/impl/BeamCalciteTable.java | 2 +- .../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 4 +- .../sql/impl/BeamSqlPipelineOptionsRegistrar.java | 2 +- .../extensions/sql/impl/BeamTableStatistics.java | 2 +- .../extensions/sql/impl/CalciteQueryPlanner.java | 2 +- .../sdk/extensions/sql/impl/JdbcConnection.java | 2 +- .../beam/sdk/extensions/sql/impl/QueryPlanner.java | 6 +- .../extensions/sql/impl/ScalarFunctionImpl.java | 4 +- .../sql/impl/UdfImplReflectiveFunctionBase.java | 2 +- .../sql/impl/parser/SqlColumnDeclaration.java | 2 +- .../sql/impl/parser/SqlCreateExternalTable.java | 2 +- .../extensions/sql/impl/parser/SqlDropObject.java | 2 +- .../extensions/sql/impl/planner/BeamRuleSets.java | 2 +- .../sql/impl/rel/BeamAggregationRel.java | 2 +- .../sdk/extensions/sql/impl/rel/BeamCalcRel.java | 2 +- .../sql/impl/rel/BeamEnumerableConverter.java | 2 +- .../sdk/extensions/sql/impl/rel/BeamIOSinkRel.java | 2 +- .../extensions/sql/impl/rel/BeamIOSourceRel.java | 2 +- .../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 4 +- .../sdk/extensions/sql/impl/rel/BeamMatchRel.java | 2 +- .../sql/impl/rel/BeamPushDownIOSourceRel.java | 4 +- .../sql/impl/rel/BeamSetOperatorRelBase.java | 2 +- .../sdk/extensions/sql/impl/rel/BeamSortRel.java | 4 +- .../sql/impl/rel/BeamTableFunctionScanRel.java | 2 +- .../extensions/sql/impl/rel/BeamUncollectRel.java | 2 +- .../sdk/extensions/sql/impl/rel/BeamUnnestRel.java | 2 +- .../sdk/extensions/sql/impl/rel/BeamValuesRel.java | 4 +- .../sql/impl/rule/BeamTableFunctionScanRule.java | 2 +- .../extensions/sql/impl/rule/BeamUnnestRule.java | 2 +- .../impl/transform/BeamBuiltinAggregations.java | 2 +- .../transform/BeamBuiltinAnalyticFunctions.java | 2 +- .../sql/impl/transform/agg/CovarianceFn.java | 2 +- .../sql/impl/utils/BigDecimalConverter.java | 2 +- .../extensions/sql/impl/utils/CalciteUtils.java | 6 +- .../sql/meta/provider/ReadOnlyTableProvider.java | 2 +- .../sql/meta/provider/bigquery/BigQueryTable.java | 2 +- .../provider/bigquery/BigQueryTableProvider.java | 2 +- .../sql/meta/provider/kafka/BeamKafkaTable.java | 2 +- .../sql/meta/provider/mongodb/MongoDbTable.java | 2 +- .../sql/meta/provider/test/TestTableProvider.java | 2 +- .../sql/meta/provider/test/TestTableUtils.java | 2 +- .../sql/meta/provider/text/TextTableProvider.java | 6 +- .../sql/meta/store/InMemoryMetaStore.java | 2 +- .../sdk/extensions/sql/BeamComplexTypeTest.java | 4 +- .../sql/BeamSqlDslSqlStdOperatorsTest.java | 8 +- .../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 2 +- .../beam/sdk/extensions/sql/BeamSqlMapTest.java | 2 +- .../extensions/sql/BeamSqlMultipleSchemasTest.java | 2 +- .../sdk/extensions/sql/PubsubToBigqueryIT.java | 4 +- .../apache/beam/sdk/extensions/sql/TestUtils.java | 2 +- .../sdk/extensions/sql/impl/JdbcDriverTest.java | 2 +- .../sql/impl/rule/JoinReorderingTest.java | 4 +- ...BeamSqlBuiltinFunctionsIntegrationTestBase.java | 6 +- .../sql/meta/CustomTableResolverTest.java | 2 +- .../provider/bigquery/BigQueryReadWriteIT.java | 2 +- .../meta/provider/bigquery/BigQueryRowCountIT.java | 2 +- .../bigquery/BigQueryTestTableProvider.java | 2 +- .../provider/kafka/BeamKafkaTableJsonTest.java | 2 +- .../provider/mongodb/MongoDbTableProviderTest.java | 2 +- .../provider/pubsub/PubsubTableProviderIT.java | 27 +--- .../meta/provider/test/TestTableProviderTest.java | 2 +- ...tTableProviderWithFilterAndProjectPushDown.java | 2 +- .../test/TestTableProviderWithFilterPushDown.java | 2 +- .../test/TestTableProviderWithProjectPushDown.java | 2 +- .../meta/provider/text/TextTableProviderTest.java | 2 +- .../beam/sdk/extensions/sql/utils/RowAsserts.java | 2 +- .../sdk/extensions/sql/zetasql/DateTimeUtils.java | 49 +++--- .../sql/zetasql/ZetaSQLQueryPlanner.java | 15 +- .../translation/ArrayScanColumnRefToUncollect.java | 4 +- .../translation/ArrayScanToJoinConverter.java | 2 +- .../zetasql/translation/ProjectScanConverter.java | 2 +- .../sql/zetasql/translation/TVFScanConverter.java | 5 +- .../zetasql/translation/TableScanConverter.java | 2 +- .../zetasql/unnest/BeamZetaSqlUncollectRel.java | 2 +- .../sql/zetasql/unnest/BeamZetaSqlUnnestRel.java | 2 +- .../sql/zetasql/ZetaSQLPushDownTest.java | 6 +- .../python/apache_beam/options/pipeline_options.py | 16 ++ .../apache_beam/runners/worker/data_sampler.py | 1 - .../apache_beam/runners/worker/sdk_worker_main.py | 58 ++++--- .../runners/worker/sdk_worker_main_test.py | 29 ++++ sdks/python/apache_beam/transforms/core.py | 90 +++++++++-- .../apache_beam/transforms/ptransform_test.py | 23 +++ .../site/content/en/contribute/release-guide.md | 14 +- 179 files changed, 2420 insertions(+), 1333 deletions(-) create mode 100644 playground/backend/cmd/migration_tool/migration_tool.go rename playground/backend/cmd/{ => remove_unused_snippets}/remove_unused_snippets.go (96%) create mode 100644 playground/backend/functions.go rename playground/backend/internal/db/{schema/migration/migration_v002.go => datastore/migration_base.go} (56%) create mode 100644 playground/backend/internal/db/datastore/migration_db.go delete mode 100644 playground/backend/internal/db/schema/migration/migrations_test.go rename playground/backend/internal/db/schema/{migration => }/migration_v001.go (74%) copy playground/backend/internal/db/{entity/schema.go => schema/migrations.go} (81%) delete mode 100644 playground/backend/internal/db/schema/version.go create mode 100644 playground/backend/internal/external_functions/external_functions_component.go create mode 100644 playground/backend/playground_functions/Dockerfile copy playground/backend/{internal/db/entity/schema.go => playground_functions/cmd/main.go} (66%) copy playground/backend/{internal/db/entity/schema.go => playground_functions/func_enviornment.go} (69%) copy playground/backend/{internal/db/entity/schema.go => playground_functions/middleware.go} (63%) copy playground/terraform/infrastructure/{ip_address => archive_file}/main.tf (80%) create mode 100644 playground/terraform/infrastructure/cloudfunctions/main.tf copy playground/terraform/infrastructure/{api_enable/variables.tf => cloudfunctions/output.tf} (68%) copy playground/terraform/infrastructure/{api_enable => cloudfunctions}/variables.tf (61%) copy playground/terraform/infrastructure/{api_enable/variables.tf => gke_bucket/main.tf} (66%) copy playground/terraform/infrastructure/{ip_address/main.tf => gke_bucket/output.tf} (89%) copy playground/terraform/infrastructure/{ip_address => gke_bucket}/variables.tf (83%) create mode 100644 release/src/main/scripts/jenkins_jobs.txt