This is an automated email from the ASF dual-hosted git repository.
scwhittle pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 7b6f9415c10 Updates Expansion Service Container to support upgrading
using the schema-transform ID (#31451)
add f93a67a6ae8 remove processing/scheduling logic from
StreamingDataflowWorker (#31317)
No new revisions were added by this update.
Summary of changes:
.../beam/runners/dataflow/worker/PubsubReader.java | 2 +-
.../beam/runners/dataflow/worker/ReaderCache.java | 4 +-
.../dataflow/worker/StreamingDataflowWorker.java | 614 +++------------------
.../worker/StreamingModeExecutionContext.java | 176 +++---
.../dataflow/worker/UngroupedWindmillReader.java | 2 +-
.../dataflow/worker/WindmillTimerInternals.java | 39 +-
.../dataflow/worker/WindowingWindmillReader.java | 2 +-
.../dataflow/worker/WorkerCustomSources.java | 2 +-
.../dataflow/worker/streaming/ActiveWorkState.java | 77 +--
.../worker/streaming/ComputationState.java | 52 +-
.../worker/streaming/ComputationStateCache.java | 1 +
.../worker/streaming/ComputationWorkExecutor.java | 118 ++++
.../{ShardedKey.java => ExecutableWork.java} | 26 +-
.../dataflow/worker/streaming/ExecutionState.java | 54 --
.../dataflow/worker/streaming/Watermarks.java | 69 +++
.../runners/dataflow/worker/streaming/Work.java | 289 ++++++----
.../runners/dataflow/worker/streaming/WorkId.java | 8 +
.../streaming/sideinput/SideInputStateFetcher.java | 2 +
.../worker/util/common/worker/MapTaskExecutor.java | 2 +-
.../client/grpc/GrpcDirectGetWorkStream.java | 53 +-
.../windmill/client/grpc/GrpcDispatcherClient.java | 32 +-
.../client/grpc/GrpcWindmillStreamFactory.java | 11 +-
.../client/grpc/StreamingEngineClient.java | 161 ++++--
.../windmill/client/grpc/WindmillStreamSender.java | 37 +-
.../worker/windmill/state/WindmillStateReader.java | 32 +-
.../windmill/work/ProcessWorkItemClient.java | 52 --
...rkItemProcessor.java => WorkItemScheduler.java} | 29 +-
.../processing/ComputationWorkExecutorFactory.java | 291 ++++++++++
.../work/processing/StreamingCommitFinalizer.java | 85 +++
.../work/processing/StreamingWorkScheduler.java | 428 ++++++++++++++
.../processing/failures/WorkFailureProcessor.java | 16 +-
.../runners/dataflow/worker/PubsubReaderTest.java | 2 +-
.../worker/StreamingDataflowWorkerTest.java | 172 +++---
.../worker/StreamingModeExecutionContextTest.java | 40 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 83 +--
.../worker/streaming/ActiveWorkStateTest.java | 213 ++++---
.../streaming/ComputationStateCacheTest.java | 45 +-
.../StreamingApplianceWorkCommitterTest.java | 19 +-
.../commits/StreamingEngineWorkCommitterTest.java | 28 +-
.../client/grpc/StreamingEngineClientTest.java | 37 +-
.../client/grpc/WindmillStreamSenderTest.java | 42 +-
.../budget/EvenGetWorkBudgetDistributorTest.java | 14 +-
.../failures/WorkFailureProcessorTest.java | 75 ++-
.../refresh/DispatchedActiveWorkRefresherTest.java | 54 +-
44 files changed, 2182 insertions(+), 1408 deletions(-)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationWorkExecutor.java
copy
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/{ShardedKey.java
=> ExecutableWork.java} (58%)
delete mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ExecutionState.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Watermarks.java
delete mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/ProcessWorkItemClient.java
rename
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/{WorkItemProcessor.java
=> WorkItemScheduler.java} (61%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/ComputationWorkExecutorFactory.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/StreamingCommitFinalizer.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/StreamingWorkScheduler.java
