This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from cba73417cd2 Move quota_project_id (#40851)
add 688a9c11da7 Fix HF Inference API test model (#40844)
add a757f49cb50 [#36841] Spark 4 streaming: Impulse, bounded reads and
PAssert on the Structured Streaming runner (#40766)
add 9c6a259d4e4 [YAML] Add SplunkIO write SchemaTransform and
WriteToSplunk (#40481)
add dcc8255806f Remove twister2 runner support (#40869)
add 21c810aed53 [InfluxDB] Wait for HTTP readiness before integration
tests (#40783)
add bd682df6697 [Dataflow Streaming] Avoid LinkedBlockingQueue.removeIf
data race in BoundedQueueExecutor (#40812)
add 5307a31bac5 Use UnsignedLong for backlog calculation and clean up
stale comments in ReadFromKafkaDoFn (#40840)
add 3afdc9aed59 Fix IcebergIO rejecting timestamp strings with a space
separator (#40861)
add 0f1ac43e906 More bound fixes
add 71133a8a632 Merge pull request #40867 from claudevdm/more-bound-fixes
add b7f6a6b1d2e [#18690] Port SolrIOTest off deprecated DoFnTester (#40819)
add 77cf5b2b5f6 Moving to 2.79.0-SNAPSHOT on master branch.
add 6178842b800 Update CHANGES.md
add 0c5adeaff12 [Spark] Resolve the structured streaming pipeline state
from the execution, cancel() returns the observed state (#40787)
add be3d44e841f Updates Delta CDC read to fail when schemas are different
at... (#40868)
No new revisions were added by this update.
Summary of changes:
.agent/skills/runners/SKILL.md | 1 -
.asf.yaml | 1 +
.github/ISSUE_TEMPLATE/bug.yml | 1 -
.github/ISSUE_TEMPLATE/failing_test.yml | 1 -
.github/ISSUE_TEMPLATE/feature.yml | 1 -
.github/ISSUE_TEMPLATE/task.yml | 1 -
.github/autolabeler.yml | 2 +-
.github/issue-rules.yml | 2 -
.../beam_PostCommit_Java_Delta_IO_Dataflow.json | 2 +-
...eam_PostCommit_Java_ValidatesRunner_Spark4.json | 2 +-
...a_ValidatesRunner_SparkStructuredStreaming.json | 3 +-
.../beam_PostCommit_Python_Dependency.json | 2 +-
.../beam_PreCommit_Java_Delta_IO_Direct.json | 2 +-
.github/workflows/README.md | 1 -
.../beam_PostCommit_Java_InfluxDbIO_IT.yml | 10 +-
...am_PostCommit_Java_ValidatesRunner_Twister2.yml | 89 ----
.../github/github_runs_prefetcher/code/config.yaml | 1 -
CHANGES.md | 47 ++-
README.md | 1 -
build.gradle.kts | 1 -
gradle.properties | 4 +-
.../main/groovy/quickstart-java-twister2.groovy | 48 ---
release/src/main/scripts/script.config | 1 -
.../dataflow/worker/util/BoundedQueueExecutor.java | 19 +-
.../io/streaming/BeamReaderCache.java | 37 +-
.../io/streaming/BeamSourceCheckpoint.java | 32 +-
.../io/streaming/UnboundedSourceDataset.java | 35 +-
.../translation/PipelineTranslatorStreaming.java | 16 +-
.../translation/StreamingEvaluationContext.java | 3 +-
.../streaming/ReadUnboundedTranslator.java | 20 +-
.../io/streaming/BeamMicroBatchSourceTest.java | 18 +-
.../io/streaming/TestUnboundedSource.java | 37 +-
.../PipelineTranslatorStreamingTest.java | 17 -
.../streaming/PAssertStreamingTest.java | 149 +++++++
.../streaming/StatefulParDoStreamingTest.java | 3 +-
.../streaming/StreamingPipelineLifecycleTest.java | 8 +-
.../SparkStructuredStreamingPipelineOptions.java | 3 +-
.../SparkStructuredStreamingPipelineResult.java | 36 +-
.../translation/SparkTransformOverrides.java | 156 +++++++
...SparkStructuredStreamingPipelineResultTest.java | 30 +-
.../StructuredStreamingPipelineStateTest.java | 52 +--
runners/twister2/build.gradle | 112 -----
.../runners/twister2/BeamBatchTSetEnvironment.java | 42 --
.../beam/runners/twister2/BeamBatchWorker.java | 166 --------
.../twister2/Twister2BatchTranslationContext.java | 44 --
.../Twister2PipelineExecutionEnvironment.java | 118 ------
.../runners/twister2/Twister2PipelineOptions.java | 77 ----
.../runners/twister2/Twister2PipelineResult.java | 89 ----
.../beam/runners/twister2/Twister2Runner.java | 360 ----------------
.../runners/twister2/Twister2RunnerRegistrar.java | 53 ---
.../twister2/Twister2StreamTranslationContext.java | 33 --
.../beam/runners/twister2/Twister2TestRunner.java | 63 ---
.../twister2/Twister2TranslationContext.java | 126 ------
.../apache/beam/runners/twister2/package-info.java | 20 -
.../wrappers/Twister2BoundedSource.java | 275 ------------
.../translation/wrappers/Twister2EmptySource.java | 41 --
.../translation/wrappers/package-info.java | 20 -
.../translators/BatchTransformTranslator.java | 29 --
.../translators/StreamTransformTranslator.java | 29 --
.../Twister2BatchPipelineTranslator.java | 94 -----
.../translators/Twister2PipelineTranslator.java | 33 --
.../Twister2StreamPipelineTranslator.java | 24 --
.../batch/AssignWindowTranslatorBatch.java | 50 ---
.../translators/batch/FlattenTranslatorBatch.java | 70 ----
.../batch/GroupByKeyTranslatorBatch.java | 80 ----
.../translators/batch/ImpulseTranslatorBatch.java | 41 --
.../batch/PCollectionViewTranslatorBatch.java | 111 -----
.../batch/ParDoMultiOutputTranslatorBatch.java | 131 ------
.../batch/ReadSourceTranslatorBatch.java | 49 ---
.../twister2/translators/batch/package-info.java | 20 -
.../functions/AssignWindowsFunction.java | 111 -----
.../translators/functions/ByteToElemFunction.java | 77 ----
.../functions/ByteToWindowFunction.java | 109 -----
.../functions/ByteToWindowFunctionPrimitive.java | 114 -----
.../translators/functions/DoFnFunction.java | 362 ----------------
.../translators/functions/ElemToBytesFunction.java | 85 ----
.../functions/GroupByWindowFunction.java | 213 ----------
.../translators/functions/ImpulseSource.java | 44 --
.../translators/functions/MapToTupleFunction.java | 110 -----
.../translators/functions/OutputTagFilter.java | 58 ---
.../functions/Twister2SinkFunction.java | 50 ---
.../internal/SystemReduceFnBuffering.java | 119 ------
.../functions/internal/package-info.java | 20 -
.../translators/functions/package-info.java | 20 -
.../runners/twister2/translators/package-info.java | 20 -
.../streaming/ReadSourceTranslatorStream.java | 32 --
.../translators/streaming/package-info.java | 20 -
.../runners/twister2/utils/NoOpStepContext.java | 36 --
.../runners/twister2/utils/TranslationUtils.java | 71 ----
.../twister2/utils/Twister2AssignContext.java | 52 ---
.../twister2/utils/Twister2SideInputReader.java | 155 -------
.../beam/runners/twister2/utils/package-info.java | 20 -
scripts/beam-sql.sh | 7 +-
sdks/go/pkg/beam/core/core.go | 2 +-
.../beam/sdk/io/delta/CreateCDCReadTasksDoFn.java | 44 +-
.../java/org/apache/beam/sdk/io/delta/DeltaIO.java | 39 +-
.../apache/beam/sdk/io/delta/DeltaSchemaUtils.java | 188 +++++++++
.../org/apache/beam/sdk/io/delta/DeltaIOTest.java | 206 +++++++++
.../beam/sdk/io/delta/DeltaSchemaUtilsTest.java | 220 ++++++++++
.../beam/sdk/io/delta/DeltaWriteTestUtils.java | 125 ++++++
sdks/java/io/expansion-service/build.gradle | 1 +
.../beam/sdk/io/iceberg/BoundAdjustment.java | 34 +-
.../apache/beam/sdk/io/iceberg/IcebergUtils.java | 42 +-
.../beam/sdk/io/iceberg/BoundAdjustmentTest.java | 81 ++++
.../beam/sdk/io/iceberg/IcebergUtilsTest.java | 58 +++
.../beam/sdk/io/iceberg/ParquetTestFiles.java | 8 +
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 16 +-
.../org/apache/beam/sdk/io/solr/SolrIOTest.java | 71 ++--
.../org/apache/beam/sdk/io/splunk/SplunkEvent.java | 8 +
.../beam/sdk/io/splunk/SplunkWriteError.java | 3 +-
.../io/splunk/SplunkWriteErrorSchemaProvider.java | 129 ++++++
.../SplunkWriteSchemaTransformConfiguration.java} | 76 ++--
.../splunk/SplunkWriteSchemaTransformProvider.java | 319 ++++++++++++++
.../beam/sdk/io/splunk/SplunkWriteErrorTest.java | 54 +++
.../SplunkWriteSchemaTransformProviderTest.java | 459 +++++++++++++++++++++
.../src/main/resources/archetype-resources/pom.xml | 37 --
.../src/main/resources/archetype-resources/pom.xml | 36 --
.../ml/transforms/embeddings/huggingface_test.py | 18 +-
.../portability/fn_api_runner/translations.py | 11 +-
.../portability/fn_api_runner/translations_test.py | 47 +++
sdks/python/apache_beam/version.py | 2 +-
sdks/python/apache_beam/yaml/integration_tests.py | 34 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 23 ++
.../yaml/test_utils/splunk_test_utils.py | 154 +++++++
sdks/python/apache_beam/yaml/tests/splunk.yaml | 60 +++
sdks/standard_external_transforms.yaml | 45 ++
sdks/typescript/package.json | 2 +-
settings.gradle.kts | 1 -
.../content/en/documentation/runners/twister2.md | 5 +-
.../site/content/en/get-started/beam-overview.md | 4 +-
website/www/site/content/en/roadmap/_index.md | 3 +-
.../layouts/partials/section-menu/en/runners.html | 2 -
132 files changed, 3051 insertions(+), 4816 deletions(-)
delete mode 100644
.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml
delete mode 100644 release/src/main/groovy/quickstart-java-twister2.groovy
create mode 100644
runners/spark/4/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/PAssertStreamingTest.java
delete mode 100644 runners/twister2/build.gradle
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/BeamBatchTSetEnvironment.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/BeamBatchWorker.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2BatchTranslationContext.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2PipelineExecutionEnvironment.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2PipelineOptions.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2PipelineResult.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2Runner.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2RunnerRegistrar.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2StreamTranslationContext.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2TestRunner.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2TranslationContext.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/package-info.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translation/wrappers/Twister2BoundedSource.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translation/wrappers/Twister2EmptySource.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translation/wrappers/package-info.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/BatchTransformTranslator.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/StreamTransformTranslator.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/Twister2BatchPipelineTranslator.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/Twister2PipelineTranslator.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/Twister2StreamPipelineTranslator.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/AssignWindowTranslatorBatch.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/FlattenTranslatorBatch.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/GroupByKeyTranslatorBatch.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/ImpulseTranslatorBatch.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/PCollectionViewTranslatorBatch.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/ParDoMultiOutputTranslatorBatch.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/ReadSourceTranslatorBatch.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/package-info.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/AssignWindowsFunction.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/ByteToElemFunction.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/ByteToWindowFunction.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/ByteToWindowFunctionPrimitive.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/DoFnFunction.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/ElemToBytesFunction.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/GroupByWindowFunction.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/ImpulseSource.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/MapToTupleFunction.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/OutputTagFilter.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/Twister2SinkFunction.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/internal/SystemReduceFnBuffering.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/internal/package-info.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/package-info.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/package-info.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/streaming/ReadSourceTranslatorStream.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/streaming/package-info.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/NoOpStepContext.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/TranslationUtils.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2AssignContext.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2SideInputReader.java
delete mode 100644
runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/package-info.java
create mode 100644
sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/DeltaSchemaUtils.java
create mode 100644
sdks/java/io/delta/src/test/java/org/apache/beam/sdk/io/delta/DeltaSchemaUtilsTest.java
create mode 100644
sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkWriteErrorSchemaProvider.java
copy
sdks/java/io/{datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogWriteSchemaTransformConfiguration.java
=>
splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkWriteSchemaTransformConfiguration.java}
(55%)
create mode 100644
sdks/java/io/splunk/src/main/java/org/apache/beam/sdk/io/splunk/SplunkWriteSchemaTransformProvider.java
create mode 100644
sdks/java/io/splunk/src/test/java/org/apache/beam/sdk/io/splunk/SplunkWriteSchemaTransformProviderTest.java
create mode 100644 sdks/python/apache_beam/yaml/test_utils/splunk_test_utils.py
create mode 100644 sdks/python/apache_beam/yaml/tests/splunk.yaml
