This is an automated email from the ASF dual-hosted git repository.
aljoscha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 513c806 [FLINK-17295] Make ExecutionAttemptID random again (#13929)
add 79b1632 [FLINK-19958] Add IOException to all I/O related Sink API
signatures
add 5ff9696 [FLINK-19758] Extract common code for file sinks into shared
module
add 1f5055c [FLINK-19758] Add unified FileSink based on new Sink API
add 97af79e [minor] Add explanatory comments to FileSinkITCase, fix
warnings
No new revisions were added by this update.
Summary of changes:
flink-connectors/flink-connector-files/pom.xml | 7 +
.../apache/flink/connector/file/sink/FileSink.java | 419 +++++++++++++++++++
.../connector/file/sink/FileSinkCommittable.java | 78 ++++
.../file/sink/FileSinkCommittableSerializer.java | 131 ++++++
.../file/sink/committer/FileCommitter.java | 64 +++
.../writer/DefaultFileWriterBucketFactory.java | 62 +++
.../connector/file/sink/writer/FileWriter.java | 281 +++++++++++++
.../file/sink/writer/FileWriterBucket.java | 322 +++++++++++++++
.../file/sink/writer/FileWriterBucketFactory.java | 50 +++
.../file/sink/writer/FileWriterBucketState.java | 97 +++++
.../writer/FileWriterBucketStateSerializer.java | 135 ++++++
.../file/sink/FileCommittableSerializerTest.java | 68 +++
.../flink/connector/file/sink/FileSinkITCase.java | 386 +++++++++++++++++
.../file/sink/committer/FileCommitterTest.java | 142 +++++++
.../file/sink/utils/FileSinkTestUtils.java | 107 +++++
.../file/sink/utils/NoOpBucketWriter.java | 62 +++
.../connector/file/sink/utils/NoOpCommitter.java | 40 +-
.../connector/file/sink/utils/NoOpRecoverable.java | 23 +-
.../utils/NoOpRecoverableFsDataOutputStream.java | 67 +++
.../file/sink/utils/NoOpRecoverableWriter.java | 75 ++++
.../FileWriterBucketStateSerializerTest.java | 78 ++++
.../file/sink/writer/FileWriterBucketTest.java | 457 +++++++++++++++++++++
.../connector/file/sink/writer/FileWriterTest.java | 235 +++++++++++
.../pom.xml | 47 +--
.../sink/filesystem/AbstractPartFileWriter.java | 0
.../functions/sink/filesystem/BucketAssigner.java | 9 +-
.../functions/sink/filesystem/BucketWriter.java | 0
.../sink/filesystem/BulkBucketWriter.java | 4 +-
.../functions/sink/filesystem/BulkPartWriter.java | 0
.../sink/filesystem/InProgressFileWriter.java | 0
.../sink/filesystem/OutputFileConfig.java | 0
.../OutputStreamBasedPartFileWriter.java | 7 +-
.../functions/sink/filesystem/PartFileInfo.java | 0
.../functions/sink/filesystem/RollingPolicy.java | 0
.../sink/filesystem/RowWiseBucketWriter.java | 4 +-
.../sink/filesystem/RowWisePartWriter.java | 0
.../sink/filesystem/WriterProperties.java | 6 +-
.../bucketassigners/BasePathBucketAssigner.java | 0
.../bucketassigners/DateTimeBucketAssigner.java | 3 +-
.../SimpleVersionedStringSerializer.java | 0
.../rollingpolicies/CheckpointRollingPolicy.java | 0
.../rollingpolicies/DefaultRollingPolicy.java | 0
.../rollingpolicies/OnCheckpointRollingPolicy.java | 0
.../DateTimeBucketAssignerTest.java | 0
.../src/test/resources/log4j2-test.properties | 0
flink-connectors/pom.xml | 1 +
.../apache/flink/api/connector/sink/Committer.java | 5 +-
.../flink/api/connector/sink/GlobalCommitter.java | 22 +-
.../org/apache/flink/api/connector/sink/Sink.java | 21 +-
.../flink/api/connector/sink/SinkWriter.java | 14 +-
flink-streaming-java/pom.xml | 6 +
.../sink/AbstractStreamingCommitterOperator.java | 5 +-
.../sink/BatchCommitterOperatorFactory.java | 17 +-
.../sink/BatchGlobalCommitterOperatorFactory.java | 17 +-
.../sink/StatelessSinkWriterOperator.java | 3 +-
.../sink/StreamingCommitterOperatorFactory.java | 21 +-
.../sink/StreamingGlobalCommitterOperator.java | 5 +-
.../StreamingGlobalCommitterOperatorFactory.java | 21 +-
.../translators/SinkTransformationTranslator.java | 106 +++--
...licyTest.java => BucketsRollingPolicyTest.java} | 2 +-
60 files changed, 3566 insertions(+), 166 deletions(-)
create mode 100644
flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/FileSink.java
create mode 100644
flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/FileSinkCommittable.java
create mode 100644
flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/FileSinkCommittableSerializer.java
create mode 100644
flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/committer/FileCommitter.java
create mode 100644
flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/writer/DefaultFileWriterBucketFactory.java
create mode 100644
flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/writer/FileWriter.java
create mode 100644
flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/writer/FileWriterBucket.java
create mode 100644
flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/writer/FileWriterBucketFactory.java
create mode 100644
flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/writer/FileWriterBucketState.java
create mode 100644
flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/writer/FileWriterBucketStateSerializer.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileCommittableSerializerTest.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkITCase.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/committer/FileCommitterTest.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/FileSinkTestUtils.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/NoOpBucketWriter.java
copy
flink-core/src/main/java/org/apache/flink/api/connector/sink/Committer.java =>
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/NoOpCommitter.java
(51%)
copy
flink-core/src/main/java/org/apache/flink/api/connector/sink/Committer.java =>
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/NoOpRecoverable.java
(51%)
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/NoOpRecoverableFsDataOutputStream.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/NoOpRecoverableWriter.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileWriterBucketStateSerializerTest.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileWriterBucketTest.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileWriterTest.java
copy flink-connectors/{flink-connector-files =>
flink-file-sink-common}/pom.xml (51%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/AbstractPartFileWriter.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssigner.java
(88%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketWriter.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkBucketWriter.java
(90%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkPartWriter.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/InProgressFileWriter.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/OutputFileConfig.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/OutputStreamBasedPartFileWriter.java
(97%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/PartFileInfo.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RollingPolicy.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWiseBucketWriter.java
(90%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWisePartWriter.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/WriterProperties.java
(89%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/BasePathBucketAssigner.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/DateTimeBucketAssigner.java
(96%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/SimpleVersionedStringSerializer.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/rollingpolicies/CheckpointRollingPolicy.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/rollingpolicies/DefaultRollingPolicy.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/rollingpolicies/OnCheckpointRollingPolicy.java
(100%)
rename {flink-streaming-java =>
flink-connectors/flink-file-sink-common}/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/bucketassigners/DateTimeBucketAssignerTest.java
(100%)
copy {flink-yarn =>
flink-connectors/flink-file-sink-common}/src/test/resources/log4j2-test.properties
(100%)
rename
flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/{RollingPolicyTest.java
=> BucketsRollingPolicyTest.java} (99%)
