This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 268ae1acf5e [IcebergIO] Support hash distribution mode when writing
rows (#38061)
add 26ef659b84a bump opentelemetry version.
add df9a0424eac add libraries
add 5223c232606 Model changes to allow OpenTelemetry context propagation
spotless. rename fields, cleanup groovy spotless rename method, fix constructor
invocation.
add 674873d7a87 Merge pull request #36962 from stankiewicz/model
add dc7b5b0190c [Gemini] Migrate all remaining uses of typing types with
built-in equivalents (#38334)
add aa5797f355a Add pipeline hash (#38357)
add 16609ed210b fix error prone. (#38372)
add efe4e941939 extend to yaml (#38371)
No new revisions were added by this update.
Summary of changes:
.github/trigger_files/beam_PostCommit_Python.json | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 8 +-
...TimeBoundedSplittableProcessElementInvoker.java | 6 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 3 +-
runners/google-cloud-dataflow-java/build.gradle | 1 +
.../beam/runners/dataflow/BatchViewOverrides.java | 6 +
.../beam/runners/dataflow/DataflowRunner.java | 3 +
.../google-cloud-dataflow-java/worker/build.gradle | 1 +
.../dataflow/worker/UngroupedWindmillReader.java | 14 +-
.../dataflow/worker/WindmillKeyedWorkItem.java | 2 +-
.../dataflow/worker/util/ValueInEmptyWindows.java | 6 +
runners/spark/spark_runner.gradle | 1 +
.../apache/beam/runners/spark/util/TimerUtils.java | 6 +
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 8 +-
.../beam/runners/dataflow/dataflowlib/execute.go | 9 +-
.../pkg/beam/runners/dataflow/dataflowlib/job.go | 12 +-
.../beam/runners/dataflow/dataflowlib/job_test.go | 51 +++++-
.../container/license_scripts/dep_urls_java.yaml | 8 +-
sdks/java/core/build.gradle | 1 +
.../apache/beam/sdk/options/SdkHarnessOptions.java | 6 +
.../org/apache/beam/sdk/transforms/DoFnTester.java | 11 +-
.../java/org/apache/beam/sdk/transforms/Reify.java | 3 +-
.../sdk/values/OpenTelemetryContextPropagator.java | 71 ++++++++
.../org/apache/beam/sdk/values/OutputBuilder.java | 3 +
.../beam/sdk/values/ValueInSingleWindow.java | 39 +++-
.../org/apache/beam/sdk/values/WindowedValue.java | 4 +
.../org/apache/beam/sdk/values/WindowedValues.java | 196 +++++++++++++++------
.../apache/beam/sdk/util/WindowedValueTest.java | 21 ++-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 3 +-
.../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 1 +
sdks/python/apache_beam/coders/coder_impl.py | 1 +
sdks/python/apache_beam/coders/coders.py | 11 +-
.../apache_beam/coders/coders_test_common.py | 3 +-
sdks/python/apache_beam/coders/observable_test.py | 3 +-
sdks/python/apache_beam/coders/row_coder_test.py | 4 +-
sdks/python/apache_beam/coders/slow_stream.py | 3 +-
.../apache_beam/coders/standard_coders_test.py | 4 +-
sdks/python/apache_beam/coders/typecoders.py | 13 +-
sdks/python/apache_beam/dataframe/frame_base.py | 3 +-
sdks/python/apache_beam/dataframe/schemas_test.py | 85 ++++-----
.../composite_transform.py | 3 +-
.../inference/tfx_bsl/build_tensorflow_model.py | 3 +-
.../apache_beam/examples/snippets/snippets_test.py | 13 +-
sdks/python/apache_beam/internal/dill_pickler.py | 6 +-
sdks/python/apache_beam/internal/metrics/metric.py | 12 +-
sdks/python/apache_beam/internal/util.py | 10 +-
sdks/python/apache_beam/io/avroio.py | 6 +-
sdks/python/apache_beam/io/avroio_test.py | 4 +-
.../apache_beam/io/components/rate_limiter.py | 4 +-
sdks/python/apache_beam/io/debezium.py | 5 +-
.../io/external/xlang_jdbcio_it_test.py | 2 +-
.../io/external/xlang_kafkaio_it_test.py | 2 +-
.../io/external/xlang_kafkaio_perf_test.py | 3 +-
.../python/apache_beam/io/filebasedio_perf_test.py | 3 +-
sdks/python/apache_beam/io/filebasedsource.py | 3 +-
sdks/python/apache_beam/io/fileio.py | 21 +--
sdks/python/apache_beam/io/filesystem.py | 8 +-
.../io/flink/flink_streaming_impulse_source.py | 3 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 39 ++--
.../apache_beam/io/gcp/bigquery_avro_tools.py | 9 +-
.../apache_beam/io/gcp/bigquery_change_history.py | 45 +++--
.../apache_beam/io/gcp/bigquery_read_internal.py | 6 +-
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 3 +-
sdks/python/apache_beam/io/gcp/bigtableio.py | 4 +-
.../apache_beam/io/gcp/datastore/v1new/helper.py | 3 +-
.../apache_beam/io/gcp/datastore/v1new/types.py | 3 +-
.../apache_beam/io/gcp/experimental/spannerio.py | 8 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 6 +-
sdks/python/apache_beam/io/iobase.py | 5 +-
sdks/python/apache_beam/io/jdbc.py | 2 +-
sdks/python/apache_beam/io/kafka.py | 6 +-
sdks/python/apache_beam/io/requestresponse.py | 19 +-
.../apache_beam/io/requestresponse_it_test.py | 3 +-
sdks/python/apache_beam/io/restriction_trackers.py | 4 +-
sdks/python/apache_beam/io/textio.py | 3 +-
sdks/python/apache_beam/metrics/cells.py | 1 +
sdks/python/apache_beam/metrics/execution.py | 1 +
sdks/python/apache_beam/metrics/metric.py | 41 ++---
sdks/python/apache_beam/metrics/metricbase.py | 3 +-
.../python/apache_beam/metrics/monitoring_infos.py | 6 +-
sdks/python/apache_beam/ml/gcp/cloud_dlp.py | 3 +-
.../python/apache_beam/ml/gcp/naturallanguageml.py | 5 +-
.../apache_beam/ml/gcp/recommendations_ai.py | 13 +-
.../apache_beam/ml/gcp/videointelligenceml.py | 3 +-
sdks/python/apache_beam/ml/gcp/visionml.py | 10 +-
.../apache_beam/ml/inference/model_manager.py | 8 +-
sdks/python/apache_beam/ml/rag/chunking/base.py | 5 +-
.../apache_beam/ml/rag/chunking/base_test.py | 3 +-
.../apache_beam/ml/rag/chunking/langchain.py | 8 +-
sdks/python/apache_beam/ml/rag/embeddings/base.py | 5 +-
.../ml/rag/enrichment/bigquery_vector_search.py | 21 +--
.../apache_beam/ml/rag/enrichment/milvus_search.py | 49 +++---
.../ml/rag/enrichment/milvus_search_it_test.py | 3 +-
.../python/apache_beam/ml/rag/ingestion/alloydb.py | 12 +-
.../ml/rag/ingestion/alloydb_it_test.py | 5 +-
.../apache_beam/ml/rag/ingestion/bigquery.py | 7 +-
.../apache_beam/ml/rag/ingestion/cloudsql.py | 22 ++-
.../ml/rag/ingestion/cloudsql_it_test.py | 3 +-
.../apache_beam/ml/rag/ingestion/jdbc_common.py | 8 +-
.../apache_beam/ml/rag/ingestion/milvus_search.py | 12 +-
sdks/python/apache_beam/ml/rag/ingestion/mysql.py | 15 +-
.../apache_beam/ml/rag/ingestion/mysql_common.py | 25 ++-
.../apache_beam/ml/rag/ingestion/postgres.py | 8 +-
.../ml/rag/ingestion/postgres_common.py | 30 ++--
.../ml/rag/ingestion/postgres_it_test.py | 5 +-
.../python/apache_beam/ml/rag/ingestion/spanner.py | 26 ++-
.../apache_beam/ml/rag/ingestion/test_utils.py | 3 +-
sdks/python/apache_beam/ml/rag/test_utils.py | 5 +-
sdks/python/apache_beam/ml/rag/types.py | 17 +-
sdks/python/apache_beam/ml/rag/utils.py | 14 +-
sdks/python/apache_beam/ml/transforms/base.py | 22 ++-
.../python/apache_beam/options/pipeline_options.py | 15 +-
sdks/python/apache_beam/options/value_provider.py | 3 +-
sdks/python/apache_beam/pipeline.py | 5 +-
sdks/python/apache_beam/pvalue.py | 3 +-
sdks/python/apache_beam/runners/common.py | 1 +
.../python/apache_beam/runners/dask/dask_runner.py | 4 +-
sdks/python/apache_beam/runners/dask/overrides.py | 16 +-
.../runners/dask/transform_evaluator.py | 8 +-
.../runners/dataflow/internal/apiclient.py | 13 +-
.../runners/dataflow/internal/apiclient_test.py | 35 ++++
.../apache_beam/runners/direct/bundle_factory.py | 5 +-
.../direct/consumer_tracking_pipeline_visitor.py | 9 +-
.../apache_beam/runners/direct/direct_runner.py | 20 +--
.../runners/direct/evaluation_context.py | 24 ++-
sdks/python/apache_beam/runners/direct/executor.py | 11 +-
.../runners/direct/transform_evaluator.py | 14 +-
.../runners/direct/watermark_manager.py | 18 +-
.../runners/interactive/augmented_pipeline.py | 6 +-
.../runners/interactive/caching/read_cache.py | 6 +-
.../runners/interactive/caching/write_cache.py | 4 +-
.../dataproc/dataproc_cluster_manager.py | 5 +-
.../runners/interactive/display/pipeline_graph.py | 11 +-
.../interactive/display/pipeline_graph_renderer.py | 3 +-
.../yaml_parse_utils.py | 12 +-
.../runners/interactive/options/capture_control.py | 5 +-
.../runners/interactive/pipeline_instrument.py | 5 +-
.../runners/interactive/sql/beam_sql_magics.py | 21 +--
.../runners/interactive/sql/sql_chain.py | 10 +-
.../apache_beam/runners/interactive/sql/utils.py | 12 +-
.../runners/interactive/testing/mock_env.py | 3 +-
.../apache_beam/runners/interactive/utils.py | 13 +-
.../python/apache_beam/runners/pipeline_context.py | 17 +-
.../runners/portability/abstract_job_service.py | 8 +-
.../runners/portability/artifact_service.py | 11 +-
.../runners/portability/expansion_service_test.py | 2 +-
.../runners/portability/fn_api_runner/execution.py | 79 ++++-----
.../runners/portability/fn_api_runner/fn_runner.py | 61 +++----
.../portability/fn_api_runner/fn_runner_test.py | 11 +-
.../portability/fn_api_runner/translations.py | 26 ++-
.../portability/fn_api_runner/trigger_manager.py | 24 ++-
.../fn_api_runner/visualization_tools.py | 6 +-
.../portability/fn_api_runner/watermark_manager.py | 19 +-
.../portability/fn_api_runner/worker_handlers.py | 6 +-
.../runners/portability/local_job_service.py | 9 +-
.../runners/portability/portable_runner.py | 8 +-
.../runners/portability/prism_runner.py | 5 +-
.../runners/portability/sdk_container_builder.py | 3 +-
.../apache_beam/runners/portability/stager.py | 32 ++--
.../apache_beam/runners/portability/stager_test.py | 3 +-
sdks/python/apache_beam/runners/sdf_utils.py | 3 +-
sdks/python/apache_beam/runners/trivial_runner.py | 5 +-
.../apache_beam/runners/worker/bundle_processor.py | 135 +++++++-------
.../apache_beam/runners/worker/data_sampler.py | 21 +--
.../runners/worker/data_sampler_test.py | 5 +-
.../apache_beam/runners/worker/log_handler.py | 3 +-
sdks/python/apache_beam/runners/worker/logger.py | 1 +
.../apache_beam/runners/worker/opcounters.py | 1 +
.../apache_beam/runners/worker/operations.py | 1 +
.../apache_beam/runners/worker/sdk_worker.py | 10 +-
.../apache_beam/runners/worker/statecache.py | 8 +-
.../apache_beam/runners/worker/statesampler.py | 3 +-
.../apache_beam/runners/worker/worker_pool_main.py | 6 +-
.../testing/analyzers/github_issues_utils.py | 12 +-
.../apache_beam/testing/analyzers/perf_analysis.py | 7 +-
.../testing/analyzers/perf_analysis_utils.py | 25 ++-
.../testing/benchmarks/nexmark/queries/query3.py | 8 +-
.../testing/load_tests/load_test_metrics_utils.py | 12 +-
.../testing/load_tests/sideinput_test.py | 8 +-
.../apache_beam/testing/synthetic_pipeline.py | 3 +-
sdks/python/apache_beam/testing/util.py | 3 +-
.../tools/runtime_type_check_microbenchmark.py | 9 +-
sdks/python/apache_beam/transforms/core.py | 4 +-
sdks/python/apache_beam/transforms/cy_combiners.py | 1 +
sdks/python/apache_beam/transforms/deduplicate.py | 4 +-
.../transforms/enrichment_handlers/cloudsql.py | 18 +-
.../python/apache_beam/transforms/external_test.py | 9 +-
.../external_transform_provider_it_test.py | 18 +-
.../apache_beam/transforms/ptransform_test.py | 98 +++++------
.../apache_beam/transforms/sideinputs_test.py | 8 +-
sdks/python/apache_beam/transforms/stats.py | 1 +
sdks/python/apache_beam/transforms/util.py | 18 +-
.../transforms/validate_runner_xlang_test.py | 15 +-
.../typehints/arrow_type_compatibility.py | 1 +
sdks/python/apache_beam/typehints/batch_test.py | 1 +
sdks/python/apache_beam/typehints/decorators.py | 1 +
.../apache_beam/typehints/decorators_test.py | 1 +
.../typehints/native_type_compatibility.py | 1 +
.../typehints/native_type_compatibility_test.py | 1 +
.../typehints/pandas_type_compatibility.py | 1 +
sdks/python/apache_beam/typehints/row_type.py | 1 +
sdks/python/apache_beam/typehints/schemas.py | 1 +
sdks/python/apache_beam/typehints/schemas_test.py | 1 +
.../python/apache_beam/typehints/typecheck_test.py | 1 +
.../apache_beam/typehints/typed_pipeline_test.py | 1 +
sdks/python/apache_beam/typehints/typehints.py | 1 +
.../python/apache_beam/typehints/typehints_test.py | 1 +
sdks/python/apache_beam/utils/counters.py | 1 +
.../apache_beam/utils/multi_process_shared.py | 3 +-
sdks/python/apache_beam/utils/proto_utils.py | 9 +-
sdks/python/apache_beam/utils/subprocess_server.py | 3 +-
sdks/python/apache_beam/utils/urns.py | 13 +-
sdks/python/apache_beam/utils/windowed_value.py | 1 +
.../yaml/examples/testing/examples_test.py | 52 +++---
sdks/python/apache_beam/yaml/yaml_io.py | 9 +-
sdks/python/apache_beam/yaml/yaml_testing.py | 7 +-
sdks/python/apache_beam/yaml/yaml_utils.py | 3 +-
sdks/python/gen_managed_doc.py | 3 +-
sdks/python/gen_xlang_wrappers.py | 20 ++-
sdks/python/ruff.toml | 2 +-
220 files changed, 1356 insertions(+), 1259 deletions(-)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/values/OpenTelemetryContextPropagator.java
