This is an automated email from the ASF dual-hosted git repository.
boyuanz pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from f137246 Merge pull request #13556 [BEAM-11472] Support SDF on
non-splitting runners such as legacy Dataflow.
new 31dcd53 [BEAM-10114] Convert PubsubLiteIO read to use SplittableDoFn.
new ec3efe2 Fix test.
new d3336fe fix: Testing issues
new 8892c1c [BEAM-10114] Convert PubsubLiteIO to use an SDF for reads.
new f5c6108 [BEAM-10114] Add byte and time limiting
new 6d8c0a9 [BEAM-10114] Add byte and time limiting
new 762b7ba [BEAM-10114] Add byte and time limiting
new 65b2c42 [BEAM-10114] Add byte and time limiting
new 418b3ac [BEAM-10114] Add byte and time limiting
new 843fc4a [BEAM-10114] Add byte and time limiting
new fa3fc2d fix: Linter issues
new ea33f9e fix: linters
new 88a3021 fix: linters
new b731da8 fix: Remove unnecessary shuffle.
new a432e8f fix: Address comments
new f87c984 Merge pull request #13470 from [BEAM-10114] Convert
PubsubLiteIO read to use SplittableDoFn.
The 29980 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
...zableSupplier.java => InitialOffsetReader.java} | 11 +-
.../io/gcp/pubsublite/InitialOffsetReaderImpl.java | 54 ++++
.../gcp/pubsublite/LimitingTopicBacklogReader.java | 81 +++++
...ffsetFinalizer.java => OffsetByteProgress.java} | 17 +-
.../io/gcp/pubsublite/OffsetByteRangeTracker.java | 172 +++++++++++
.../io/gcp/pubsublite/OffsetCheckpointMark.java | 74 -----
.../pubsublite/PerSubscriptionPartitionSdf.java | 110 +++++++
.../beam/sdk/io/gcp/pubsublite/PublisherCache.java | 2 +-
.../beam/sdk/io/gcp/pubsublite/PubsubLiteIO.java | 14 +-
.../gcp/pubsublite/PubsubLiteUnboundedReader.java | 333 ---------------------
.../gcp/pubsublite/PubsubLiteUnboundedSource.java | 126 --------
...zer.java => SerializableSubscriberFactory.java} | 17 +-
.../io/gcp/pubsublite/SerializableSupplier.java | 2 +-
.../sdk/io/gcp/pubsublite/SubscribeTransform.java | 124 ++++++++
.../sdk/io/gcp/pubsublite/SubscriberOptions.java | 42 ++-
...etFinalizer.java => SubscriptionPartition.java} | 19 +-
...dCoder.java => SubscriptionPartitionCoder.java} | 36 ++-
...er.java => SubscriptionPartitionProcessor.java} | 14 +-
... => SubscriptionPartitionProcessorFactory.java} | 14 +-
.../SubscriptionPartitionProcessorImpl.java | 152 ++++++++++
.../sdk/io/gcp/pubsublite/TopicBacklogReader.java | 33 +-
.../io/gcp/pubsublite/TopicBacklogReaderImpl.java | 80 ++---
.../gcp/pubsublite/TopicBacklogReaderSettings.java | 31 +-
.../gcp/pubsublite/TranslatingPullSubscriber.java | 55 ----
.../beam/sdk/io/gcp/pubsublite/UuidCoder.java | 7 +-
.../gcp/pubsublite/OffsetByteRangeTrackerTest.java | 158 ++++++++++
.../gcp/pubsublite/OffsetCheckpointMarkTest.java | 81 -----
.../PerSubscriptionPartitionSdfTest.java | 189 ++++++++++++
.../pubsublite/PubsubLiteUnboundedReaderTest.java | 330 --------------------
.../SubscriptionPartitionProcessorImplTest.java | 227 ++++++++++++++
.../gcp/pubsublite/TopicBacklogReaderImplTest.java | 129 ++------
31 files changed, 1493 insertions(+), 1241 deletions(-)
copy
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{SerializableSupplier.java
=> InitialOffsetReader.java} (80%)
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/InitialOffsetReaderImpl.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/LimitingTopicBacklogReader.java
copy
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java
=> OffsetByteProgress.java} (68%)
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetByteRangeTracker.java
delete mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetCheckpointMark.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PerSubscriptionPartitionSdf.java
delete mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteUnboundedReader.java
delete mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteUnboundedSource.java
copy
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java
=> SerializableSubscriberFactory.java} (65%)
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscribeTransform.java
copy
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java
=> SubscriptionPartition.java} (64%)
copy
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{UuidCoder.java
=> SubscriptionPartitionCoder.java} (50%)
rename
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java
=> SubscriptionPartitionProcessor.java} (74%)
copy
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{SerializableSupplier.java
=> SubscriptionPartitionProcessorFactory.java} (60%)
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionProcessorImpl.java
delete mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/TranslatingPullSubscriber.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetByteRangeTrackerTest.java
delete mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetCheckpointMarkTest.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/PerSubscriptionPartitionSdfTest.java
delete mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteUnboundedReaderTest.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionProcessorImplTest.java
