This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 14e8de6e99a Fix typo. (#21864)
add 634b0453469 Standardizing naming and URN for Pubsub Read Schema
Transform (#25170)
add 05dc71a610f Attempt fix GCPIO_Direct tests timeout (#25209)
add b2d500f8494 Fix pulling licenses (#25234)
add f77366a8115 Ignore flags for beam_sql magic (#25210)
add cfe8005c502 Deterministic ordering of gbk outputs for testing.
add 3f825c39106 Increase timeouts for cross-langauge tests.
add 99999c6766b Merge pull request #25241 Fix typescript github action
failures.
add 9a6354879dd Stop publishing empty test-only artifacts (#25191)
add 957301519bb Fix Debezium expansion service fail to start (#25243)
add bc2895c99a2 Merge pull request #25094: Externalizing the StreamWriter
parameters for StorageWrites
add aae7ecd2441 Setup typedoc for doc generation.
add 9e373713cf8 Quick pass adding some typescript docs and pointers.
add 68b60deefa3 Add typescript doc gen to release process.
add d182f462a5d Split README into user-facing and dev-facing portions.
add 3883affd24e Merge branch 'master' into typescript-typedoc
add 8f7d2cc8eb9 Merge pull request #25069 Run typedoc to generate
typescript docs.
add b17c1c66921 Update/add torch versions to tox.ini (#25045)
add 7a6598f2c96 Add support for templates in task hints (#25214)
add af416a4cc4f Bump torch (#25057)
add fde7909c951 Fix XVR_Direct time out (#25247)
add f61c3fd32c2 Exclude changestream integration test on
DataflowLegacyWorker (#25239)
add 067e4db747b [Tour of Beam] [Task] Fix dependency management for 2.44
Playground java runner (#25230)
add d7264d4ba2a Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#25270)
add 01aa470cf72 [Spark Runner] Add new experiment that provides concurrent
bounded output for SDFs (resolves #23852) (#24837)
add 10805a2d402 Add sideinputs to the RunInference Transform (#25200)
add 74de85513b9 Move changes to correct release in CHANGES.md (#25288)
add 3f8b8e383d7 Pass instead of raising an error (#25287)
add 24f40aab7aa [WebSite] Add new Python quickstart (#24804)
add 49f00ab5c8d [CdapIO] Implement windowed write (#25206)
No new revisions were added by this update.
Summary of changes:
.github/workflows/build_playground_backend.yml | 9 -
.github/workflows/playground_deploy_backend.yml | 9 -
.../workflows/playground_examples_ci_reusable.yml | 6 +-
.github/workflows/typescript_tests.yml | 2 +-
...mmit_CrossLanguageValidatesRunner_Direct.groovy | 4 +-
...ommit_CrossLanguageValidatesRunner_Flink.groovy | 2 +
...ommit_CrossLanguageValidatesRunner_Samza.groovy | 2 +
...ommit_CrossLanguageValidatesRunner_Spark.groovy | 2 +
...mmit_CrossLanguageValidatesRunner_Spark3.groovy | 2 +
CHANGES.md | 35 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 14 +-
.../backend/internal/fs_content/load.go | 6 +-
.../backend/internal/fs_content/load_test.go | 2 +-
.../backend/samples/api/get_unit_content.json | 2 +-
.../module 1/group/unit-challenge/hint1.md | 2 +-
.../module 1/unit-challenge/hint1.md | 2 +-
.../module 2/unit-challenge/hint1.md | 2 +-
playground/backend/containers/java/Dockerfile | 55 ++-
playground/backend/containers/java/build.gradle | 8 +-
.../src/main/scripts/build_release_candidate.sh | 15 +
.../construction/SplittableParDoNaiveBounded.java | 3 +-
runners/google-cloud-dataflow-java/build.gradle | 6 +-
runners/spark/spark_runner.gradle | 41 ++-
.../spark/translation/MultiDoFnFunction.java | 66 ++--
.../spark/translation/SparkInputDataProcessor.java | 374 +++++++++++++++++++++
.../spark/translation/SparkProcessContext.java | 149 ++------
.../spark/translation/TransformTranslator.java | 16 +-
.../streaming/StreamingTransformTranslator.java | 3 +-
.../translation/SparkInputDataProcessorTest.java | 278 +++++++++++++++
sdks/go.mod | 6 +-
sdks/go.sum | 12 +-
sdks/java/extensions/sql/perf-tests/build.gradle | 1 +
.../java/org/apache/beam/sdk/io/cdap/CdapIO.java | 44 ++-
.../org/apache/beam/sdk/io/cdap/CdapIOTest.java | 36 ++
.../DebeziumReadSchemaTransformProvider.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 13 +
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 10 +
.../gcp/bigquery/WriteGroupedRecordsToFiles.java | 3 +-
...=> PubsubReadSchemaTransformConfiguration.java} | 10 +-
...java => PubsubReadSchemaTransformProvider.java} | 32 +-
.../PubsubSchemaTransformMessageToRowFactory.java | 18 +-
.../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 3 +
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 27 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 2 +
... => PubsubReadSchemaTransformProviderTest.java} | 62 ++--
...bsubSchemaTransformMessageToRowFactoryTest.java | 28 +-
.../SpannerChangeStreamErrorTest.java | 3 +
sdks/java/testing/expansion-service/build.gradle | 1 +
sdks/java/testing/kafka-service/build.gradle | 4 +-
.../online_clustering/clustering_pipeline/setup.py | 2 +-
.../inference/run_inference_side_inputs.py | 165 +++++++++
.../transforms/elementwise/runinference_test.py | 45 +--
sdks/python/apache_beam/ml/inference/base.py | 161 +++++++--
sdks/python/apache_beam/ml/inference/base_test.py | 177 +++++++++-
.../apache_beam/ml/inference/pytorch_inference.py | 77 ++---
.../ml/inference/pytorch_inference_test.py | 5 +-
.../apache_beam/ml/inference/sklearn_inference.py | 37 +-
.../apache_beam/ml/inference/tensorrt_inference.py | 13 +-
sdks/python/apache_beam/ml/inference/utils.py | 48 +++
.../apache_beam/runners/interactive/sql/utils.py | 5 +-
.../container/license_scripts/dep_urls_py.yaml | 2 +-
sdks/python/test-suites/direct/common.gradle | 2 +-
sdks/python/test-suites/tox/py38/build.gradle | 12 +
sdks/python/tox.ini | 5 +-
sdks/typescript/{README.md => README-dev.md} | 89 +----
sdks/typescript/README.md | 152 ++-------
sdks/typescript/package-lock.json | 177 +++++++++-
sdks/typescript/package.json | 4 +-
sdks/typescript/src/apache_beam/coders/coders.ts | 1 +
.../typescript/src/apache_beam/coders/js_coders.ts | 7 +
.../typescript/src/apache_beam/coders/row_coder.ts | 20 +-
.../src/apache_beam/examples/wordcount_textio.ts | 2 +-
sdks/typescript/src/apache_beam/pvalue.ts | 9 +-
.../src/apache_beam/runners/direct_runner.ts | 2 +-
sdks/typescript/src/apache_beam/runners/runner.ts | 16 +-
sdks/typescript/src/apache_beam/testing/assert.ts | 23 ++
.../apache_beam/testing/multi_pipeline_runner.ts | 19 ++
.../src/apache_beam/transforms/external.ts | 9 +
.../src/apache_beam/transforms/flatten.ts | 7 +
.../apache_beam/transforms/group_and_combine.ts | 40 ++-
.../src/apache_beam/transforms/internal.ts | 14 +
.../typescript/src/apache_beam/transforms/pardo.ts | 62 +++-
.../src/apache_beam/transforms/python.ts | 35 ++
.../src/apache_beam/transforms/transform.ts | 50 +++
.../src/apache_beam/transforms/window.ts | 8 +
sdks/typescript/src/apache_beam/utils/utils.ts | 2 +
sdks/typescript/test/primitives_test.ts | 14 +-
sdks/typescript/test/wordcount.ts | 6 +-
sdks/typescript/typedoc.json | 24 ++
.../site/content/en/get-started/quickstart-py.md | 31 +-
.../content/en/get-started/quickstart/python.md | 226 +++++++++++++
.../partials/section-menu/en/get-started.html | 8 +-
92 files changed, 2488 insertions(+), 759 deletions(-)
create mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkInputDataProcessor.java
create mode 100644
runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SparkInputDataProcessorTest.java
rename
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/{PubsubSchemaTransformReadConfiguration.java
=> PubsubReadSchemaTransformConfiguration.java} (94%)
rename
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/{PubsubSchemaTransformReadProvider.java
=> PubsubReadSchemaTransformProvider.java} (88%)
rename
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/{PubsubSchemaTransformReadProviderTest.java
=> PubsubReadSchemaTransformProviderTest.java} (87%)
create mode 100644
sdks/python/apache_beam/examples/inference/run_inference_side_inputs.py
create mode 100644 sdks/python/apache_beam/ml/inference/utils.py
copy sdks/typescript/{README.md => README-dev.md} (51%)
create mode 100644 sdks/typescript/typedoc.json
create mode 100644 website/www/site/content/en/get-started/quickstart/python.md
