This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a change to branch users/damccorm/prismByDefault
in repository https://gitbox.apache.org/repos/asf/beam.git
from 20370cb51f0 exclude unsupported state
add 61cf27fdef5 [IcebergIO] Apply delete/update filter when reading
(#34607)
add bd47dc90c37 Use component coder when handling nullable coder in prism.
(#34615)
add 76708d54438 added ddl to SqlTransform (#34614)
add e646bc30729 use the script when validating the BQ table (#34617)
add 7cb43f23d37 Fix GCP auth step (#34624)
add 80c0998ff7a Use cached prism binary without re-unzipping by default.
(#34616)
add bd2891db827 Update run_rc_validation_python_mobile_gaming.yml (#34620)
add 18373bfc549 Preserve pane index through reshuffle. (#34348)
add 5a590ad513b Revert "Add logic to fail if specified partitions do not
exist in the topic. …" (#34631)
add a699e8e03f0 Fix The PostCommit XVR Flink job (#34628)
add 3af039802ee Bump github.com/nats-io/nats-server/v2 from 2.11.0 to
2.11.1 in /sdks (#34586)
add c1650e9cc74 Normalize tf record io (#34411)
add de2a83f3673 Bump github.com/docker/docker in /sdks (#34483)
add bebfcbc84bd Add an experiment to make prism server a singleton (#34623)
add 458d7a72fa5 Fix new flaky postcommit python tests (#34638)
add cbaa083f415 Merge branch 'master' of https://github.com/apache/beam
into users/damccorm/prismByDefault
No new revisions were added by this update.
Summary of changes:
.../IO_Iceberg_Integration_Tests.json | 2 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 9 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 8 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 9 +-
.github/workflows/refresh_looker_metrics.yml | 9 +-
.../republish_released_docker_containers.yml | 8 +-
.../run_rc_validation_python_mobile_gaming.yml | 110 +---
.../apache/beam/runners/flink/FlinkJobInvoker.java | 17 +-
.../beam/runners/jobsubmission/JobInvoker.java | 5 +-
runners/prism/java/build.gradle | 2 -
scripts/tools/validate_table.sh | 117 ++++
sdks/go.mod | 4 +-
sdks/go.sum | 8 +-
sdks/go/pkg/beam/runners/prism/internal/coders.go | 12 +-
.../TFRecordReadSchemaTransformConfiguration.java | 104 ++++
.../io/TFRecordReadSchemaTransformProvider.java | 201 +++++++
.../io/TFRecordSchemaTransformTranslation.java} | 30 +-
.../TFRecordWriteSchemaTransformConfiguration.java | 107 ++++
.../io/TFRecordWriteSchemaTransformProvider.java | 244 +++++++++
.../io/TFRecordSchemaTransformProviderTest.java | 587 +++++++++++++++++++++
.../expansion/ExternalSqlTransformRegistrar.java | 9 +
.../apache/beam/sdk/io/iceberg/ScanTaskReader.java | 5 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 10 +-
.../beam/sdk/io/kafka/KafkaUnboundedSource.java | 34 +-
.../sdk/io/kafka/WatchForKafkaTopicPartitions.java | 10 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 98 ----
.../examples/streaming_wordcount_it_test.py | 2 +-
.../apache_beam/io/gcp/bigquery_read_it_test.py | 3 +
.../apache_beam/io/gcp/bigquery_write_it_test.py | 4 +-
.../apache_beam/io/gcp/gcsio_integration_test.py | 15 +-
.../runners/portability/prism_runner.py | 67 ++-
.../runners/portability/prism_runner_test.py | 199 ++++++-
sdks/python/apache_beam/testing/util.py | 20 +-
sdks/python/apache_beam/transforms/sql.py | 9 +-
sdks/python/apache_beam/transforms/sql_test.py | 24 +
sdks/python/apache_beam/transforms/util.py | 78 ++-
sdks/python/apache_beam/transforms/util_test.py | 220 +++++++-
sdks/python/apache_beam/yaml/json_utils.py | 1 +
sdks/python/apache_beam/yaml/standard_io.yaml | 40 +-
.../yaml/tests/{csv.yaml => tfrecord_write.yaml} | 39 +-
sdks/python/apache_beam/yaml/yaml_io.py | 84 +++
sdks/python/apache_beam/yaml/yaml_mapping.py | 2 +
sdks/standard_external_transforms.yaml | 62 ++-
43 files changed, 2272 insertions(+), 356 deletions(-)
create mode 100755 scripts/tools/validate_table.sh
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordReadSchemaTransformConfiguration.java
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordReadSchemaTransformProvider.java
copy
sdks/java/{io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQuerySchemaTransformTranslation.java
=>
core/src/main/java/org/apache/beam/sdk/io/TFRecordSchemaTransformTranslation.java}
(69%)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordWriteSchemaTransformConfiguration.java
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordWriteSchemaTransformProvider.java
create mode 100644
sdks/java/core/src/test/java/org/apache/beam/sdk/io/TFRecordSchemaTransformProviderTest.java
copy sdks/python/apache_beam/yaml/tests/{csv.yaml => tfrecord_write.yaml} (54%)
