This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from b3a5df21e1c Bump golang.org/x/net in /learning/tour-of-beam/backend
(#39984)
add 6e33c516a40 Update Java BOM
add 756af46efe9 Merge pull request #39954 from apache/upgrade-bom-sep-1
add d201c6feb7c [CI] Re-pin gradle/actions/setup-gradle to allowlisted
v6.3.0
add 332ef68ed22 Merge pull request #39986 from
tkaymak/ci-setup-gradle-allowlist
add 562273c38b0 Bump zizmorcore/zizmor-action from 0.6.2 to 0.6.3 (#39968)
add 9accecd14f9 Fix nullness in BigQueryIO
add b20e5fd10af Merge pull request #39893: Fix nullness in BigQueryIO
add 49c5f2271df Update staticcheck (#39987)
add ffcbd7e464f Update IO support matrix after Beam 2.76.0 release (#39982)
add cdaa5ff456d [Java] Remove MS_EXPOSE_REP from the global SpotBugs
filter (#39951)
add bddab868f3c ValueKind coder (#39989)
add f5d35728554 [IcebergIO] Spec aware file reserialization (#39973)
add 496087bb5eb AddFiles: tighten file schemas (#39975)
add c5a2eac1178 [IcebergIO] CDC record and sort key (#39981)
No new revisions were added by this update.
Summary of changes:
.../actions/setup-environment-action/action.yml | 2 +-
.github/trigger_files/beam_PostCommit_Java.json | 1 +
.../beam_PostCommit_Java_BigQueryEarlyRollout.json | 3 +
.../beam_PostCommit_Java_DataflowV1.json | 1 +
.../beam_PostCommit_Java_DataflowV2.json | 1 +
.github/workflows/beam_PreCommit_GHA.yml | 2 +-
.github/workflows/go_tests.yml | 4 +-
.../republish_released_docker_containers.yml | 4 +-
CHANGES.md | 10 +-
.../runners/spark/metrics/MetricsAccumulator.java | 5 +
.../metrics/MetricsAccumulator.java | 5 +
.../src/main/resources/beam/spotbugs-filter.xml | 1 -
.../org/apache/beam/sdk/coders/CoderRegistry.java | 3 +
.../org/apache/beam/sdk/coders/ValueKindCoder.java | 88 +++++
.../java/org/apache/beam/sdk/metrics/Lineage.java | 9 +
.../org/apache/beam/sdk/metrics/SourceMetrics.java | 9 +
.../beam/sdk/util/construction/ModelCoders.java | 7 +
.../util/construction/PTransformTranslation.java | 7 +
...tegerCoderTest.java => ValueKindCoderTest.java} | 49 +--
.../AvroGenericRecordToStorageApiProto.java | 22 +-
.../beam/sdk/io/gcp/bigquery/AvroRowWriter.java | 9 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 17 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 17 +-
...ueryExportReadSchemaTransformConfiguration.java | 8 +-
.../BigQueryExportReadSchemaTransformProvider.java | 13 +-
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 38 ++-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 373 ++++++++++++---------
.../sdk/io/gcp/bigquery/BigQueryIOTranslation.java | 269 +++++++++------
.../io/gcp/bigquery/BigQueryQuerySourceDef.java | 32 +-
.../io/gcp/bigquery/BigQuerySchemaIOProvider.java | 7 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 49 ++-
.../io/gcp/bigquery/BigQueryStorageAvroReader.java | 12 +-
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 169 +++++-----
.../gcp/bigquery/DynamicDestinationsHelpers.java | 12 +-
.../beam/sdk/io/gcp/bigquery/RowWriterFactory.java | 7 +-
.../io/gcp/bigquery/StorageApiConvertMessages.java | 7 +-
.../StorageApiDynamicDestinationsBeamRow.java | 1 -
...StorageApiDynamicDestinationsGenericRecord.java | 4 +-
.../StorageApiDynamicDestinationsProto.java | 12 +-
.../gcp/bigquery/StorageApiFinalizeWritesDoFn.java | 6 -
.../beam/sdk/io/gcp/bigquery/StorageApiLoads.java | 17 +-
.../io/gcp/bigquery/StorageApiWritePayload.java | 1 -
.../bigquery/StorageApiWritesShardedRecords.java | 4 +-
.../beam/sdk/io/gcp/bigquery/StreamingInserts.java | 4 +-
.../io/gcp/bigquery/TableRowToStorageApiProto.java | 11 +-
.../io/gcp/bigquery/UpdateSchemaDestination.java | 26 +-
.../BigQueryDirectReadSchemaTransformProvider.java | 32 +-
.../BigQueryFileLoadsSchemaTransformProvider.java | 27 +-
...ueryStorageWriteApiSchemaTransformProvider.java | 102 +++---
.../providers/BigQueryWriteConfiguration.java | 18 +
.../providers/PortableBigQueryDestinations.java | 6 +-
sdks/java/io/iceberg/build.gradle | 1 +
.../sdk/io/iceberg/CollectDistinctSchemas.java | 198 +++++++++--
.../apache/beam/sdk/io/iceberg/FileSchemas.java | 201 ++++++++++-
.../beam/sdk/io/iceberg/ReadFooterSchema.java | 38 +--
.../sdk/io/iceberg/SerializableDeleteFile.java | 85 ++++-
.../io/iceberg/cdc/SerializableChangelogTask.java | 10 +-
.../beam/sdk/io/iceberg/cdc/sink/CdcRecord.java | 84 +++++
.../sdk/io/iceberg/cdc/sink/CdcRecordCoder.java | 99 ++++++
.../beam/sdk/io/iceberg/cdc/sink/CdcSortKey.java | 77 +++++
.../sdk/io/iceberg/cdc/sink}/package-info.java | 5 +-
.../sdk/io/iceberg/CollectDistinctSchemasTest.java | 107 ++++--
.../beam/sdk/io/iceberg/FileSchemasTest.java | 345 +++++++++++++++++++
.../beam/sdk/io/iceberg/ReadFooterSchemaTest.java | 49 ++-
.../sdk/io/iceberg/SerializableDeleteFileTest.java | 281 ++++++++++++++--
.../beam/sdk/io/iceberg/cdc/CdcReadUtilsTest.java | 5 +-
.../sdk/io/iceberg/cdc/ReadFromChangelogsTest.java | 5 +-
.../io/iceberg/cdc/sink/CdcRecordCoderTest.java | 113 +++++++
.../sdk/io/iceberg/cdc/sink/CdcSortKeyTest.java | 230 +++++++++++++
.../apache/beam/sdk/io/thrift/ThriftSchema.java | 5 +
.../apache/beam/sdk/testutils/NamedTestResult.java | 7 +
.../org/apache/beam/sdk/tpcds/TpcdsSchemas.java | 6 +
sdks/python/build.gradle | 8 +
website/www/site/content/en/blog/beam-2.76.0.md | 4 -
.../site/content/en/documentation/io/connectors.md | 181 ++++------
website/www/site/data/capability_matrix.yaml | 1 -
76 files changed, 2842 insertions(+), 846 deletions(-)
create mode 100644
.github/trigger_files/beam_PostCommit_Java_BigQueryEarlyRollout.json
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/coders/ValueKindCoder.java
copy
sdks/java/core/src/test/java/org/apache/beam/sdk/coders/{BigEndianIntegerCoderTest.java
=> ValueKindCoderTest.java} (58%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecord.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecordCoder.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcSortKey.java
copy {it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dlp =>
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink}/package-info.java
(80%)
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecordCoderTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcSortKeyTest.java
