This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 71c8459633e [Blog Post] Apache Beam for a content discovery platform
(#28734)
add 1c5982567b3 Bump cloud.google.com/go/bigtable from 1.19.0 to 1.20.0 in
/sdks (#28789)
add 25024064ed2 Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#28790)
add b556aee9177 Bump urllib3 from 1.26.16 to 1.26.17 in
/sdks/python/container/py38 (#28787)
add c7d7896533b Github Workflow Replacement for Jenkins Jobs,
beam_LoadTests_Java_GBK_Dataflow_Batch_* (#28738)
add 2e0521162f6 [Java BQ] Storage API streaming load test (#28264)
add 3655b1ea3fb Update dyi-content-discovery-platform-genai-beam.md
add f3df03d8fa9 Merge pull request #28788 from apache/aaltay-patch-1
add 12b07140203 Add notebook for per key models (#28327)
add c01b41f9758 Bump urllib3 from 1.26.16 to 1.26.17 in
/sdks/python/container/py310 (#28785)
add be805372238 BigQuery testing suite that runs against BQ's day 0 region
(#28397)
add f6b43ce1736 Revert "assign highmem runner to beam_PostCommit_Python
and to beam_PreCommit_Java_GCP_IO_Direct (#28719)" (#28804)
No new revisions were added by this update.
Summary of changes:
.../beam_LoadTests_Java_GBK_Dataflow_Batch.yml | 141 +++++
.../beam_LoadTests_Java_GBK_Dataflow_Streaming.yml | 141 +++++
...LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml | 166 ++++++
...LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml | 168 ++++++
...Tests_Java_GBK_Dataflow_V2_Streaming_Java11.yml | 166 ++++++
...Tests_Java_GBK_Dataflow_V2_Streaming_Java17.yml | 168 ++++++
... beam_PostCommit_Java_BigQueryEarlyRollout.yml} | 68 +--
.../beam_PostCommit_Java_IO_Performance_Tests.yml | 2 +-
.github/workflows/beam_PostCommit_Python.yml | 2 +-
.../beam_PreCommit_Java_GCP_IO_Direct.yml | 2 +-
...java_GBK_Dataflow_Batch_2GB_of_100B_records.txt | 28 +
...ava_GBK_Dataflow_Batch_2GB_of_100kB_records.txt | 28 +
.../java_GBK_Dataflow_Batch_2GB_of_10B_records.txt | 28 +
...nout_4_times_with_2GB_10-byte_records_total.txt | 28 +
...nout_8_times_with_2GB_10-byte_records_total.txt | 28 +
...ataflow_Batch_reiterate_4_times_10kB_values.txt | 28 +
...Dataflow_Batch_reiterate_4_times_2MB_values.txt | 28 +
..._GBK_Dataflow_Streaming_2GB_of_100B_records.txt | 29 +
...GBK_Dataflow_Streaming_2GB_of_100kB_records.txt | 29 +
...a_GBK_Dataflow_Streaming_2GB_of_10B_records.txt | 29 +
...nout_4_times_with_2GB_10-byte_records_total.txt | 29 +
...nout_8_times_with_2GB_10-byte_records_total.txt | 29 +
...low_Streaming_reiterate_4_times_10kB_values.txt | 29 +
...flow_Streaming_reiterate_4_times_2MB_values.txt | 29 +
...ataflow_V2_Batch_Java11_2GB_of_100B_records.txt | 29 +
...taflow_V2_Batch_Java11_2GB_of_100kB_records.txt | 29 +
...Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt | 29 +
...nout_4_times_with_2GB_10-byte_records_total.txt | 29 +
...nout_8_times_with_2GB_10-byte_records_total.txt | 29 +
..._Batch_Java11_reiterate_4_times_10kB_values.txt | 29 +
...2_Batch_Java11_reiterate_4_times_2MB_values.txt | 29 +
...ataflow_V2_Batch_Java17_2GB_of_100B_records.txt | 29 +
...taflow_V2_Batch_Java17_2GB_of_100kB_records.txt | 29 +
...Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt | 29 +
...nout_4_times_with_2GB_10-byte_records_total.txt | 29 +
...nout_8_times_with_2GB_10-byte_records_total.txt | 29 +
..._Batch_Java17_reiterate_4_times_10kB_values.txt | 29 +
...2_Batch_Java17_reiterate_4_times_2MB_values.txt | 29 +
...low_V2_Streaming_Java11_2GB_of_100B_records.txt | 30 +
...ow_V2_Streaming_Java11_2GB_of_100kB_records.txt | 30 +
...flow_V2_Streaming_Java11_2GB_of_10B_records.txt | 30 +
...nout_4_times_with_2GB_10-byte_records_total.txt | 30 +
...nout_8_times_with_2GB_10-byte_records_total.txt | 30 +
...eaming_Java11_reiterate_4_times_10kB_values.txt | 30 +
...reaming_Java11_reiterate_4_times_2MB_values.txt | 30 +
...low_V2_Streaming_Java17_2GB_of_100B_records.txt | 30 +
...ow_V2_Streaming_Java17_2GB_of_100kB_records.txt | 30 +
...flow_V2_Streaming_Java17_2GB_of_10B_records.txt | 30 +
...nout_4_times_with_2GB_10-byte_records_total.txt | 30 +
...nout_8_times_with_2GB_10-byte_records_total.txt | 30 +
...eaming_Java17_reiterate_4_times_10kB_values.txt | 30 +
...reaming_Java17_reiterate_4_times_2MB_values.txt | 30 +
examples/notebooks/beam-ml/README.md | 1 +
examples/notebooks/beam-ml/per_key_models.ipynb | 608 +++++++++++++++++++
it/build.gradle | 4 +
it/google-cloud-platform/build.gradle | 5 +-
.../beam/it/gcp/bigquery/BigQueryStreamingLT.java | 643 +++++++++++++++++++++
sdks/go.mod | 10 +-
sdks/go.sum | 16 +-
sdks/java/io/google-cloud-platform/build.gradle | 44 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 6 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 6 +-
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 29 +-
.../sdk/io/gcp/bigquery/TestBigQueryOptions.java | 7 +
.../gcp/common/GcpIoPipelineOptionsRegistrar.java | 2 +
.../beam/sdk/io/gcp/testing/BigqueryClient.java | 37 +-
.../io/gcp/bigquery/BigQueryIOStorageQueryIT.java | 10 +-
.../io/gcp/bigquery/BigQueryIOStorageReadIT.java | 9 +-
.../bigquery/BigQueryIOStorageReadTableRowIT.java | 43 +-
.../io/gcp/bigquery/BigQueryIOStorageWriteIT.java | 31 +-
.../bigquery/BigQuerySchemaUpdateOptionsIT.java | 6 +-
.../BigQueryTimePartitioningClusteringIT.java | 42 +-
.../sdk/io/gcp/bigquery/BigQueryToTableIT.java | 9 +-
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 37 +-
.../sdk/io/gcp/bigquery/FileLoadsStreamingIT.java | 9 +-
.../bigquery/StorageApiDirectWriteProtosIT.java | 9 +-
.../gcp/bigquery/StorageApiSinkFailedRowsIT.java | 13 +-
.../io/gcp/bigquery/StorageApiSinkRowUpdateIT.java | 9 +-
.../gcp/bigquery/StorageApiSinkSchemaUpdateIT.java | 17 +-
.../gcp/bigquery/TableRowToStorageApiProtoIT.java | 11 +-
.../container/py310/base_image_requirements.txt | 2 +-
.../container/py38/base_image_requirements.txt | 2 +-
.../dyi-content-discovery-platform-genai-beam.md | 2 +-
83 files changed, 3802 insertions(+), 126 deletions(-)
create mode 100644 .github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml
create mode 100644
.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml
create mode 100644
.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml
create mode 100644
.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml
create mode 100644
.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml
create mode 100644
.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml
copy .github/workflows/{beam_PostCommit_Python.yml =>
beam_PostCommit_Java_BigQueryEarlyRollout.yml} (57%)
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_100kB_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_2GB_of_10B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_10kB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Batch_reiterate_4_times_2MB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_100kB_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_2GB_of_10B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_10kB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_Streaming_reiterate_4_times_2MB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_100kB_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_2GB_of_10B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_10kB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Batch_Java17_reiterate_4_times_2MB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_100kB_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_2GB_of_10B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_4_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_fanout_8_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_10kB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_GBK_Dataflow_V2_Streaming_Java17_reiterate_4_times_2MB_values.txt
create mode 100644 examples/notebooks/beam-ml/per_key_models.ipynb
create mode 100644
it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java
