(flink-cdc) branch master updated: [FLINK-35127][cdc][values] Remove HybridSource from ValuesSource to avoid CI failure. (#3237)
This is an automated email from the ASF dual-hosted git repository.
renqs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
The following commit(s) were added to refs/heads/master by this push:
new d4ed7db8d [FLINK-35127][cdc][values] Remove HybridSource from
ValuesSource to avoid CI failure. (#3237)
d4ed7db8d is described below
commit d4ed7db8dca18906bb8a1d7738cb158da0222bc3
Author: Kunni
AuthorDate: Fri Apr 19 15:37:35 2024 +0800
[FLINK-35127][cdc][values] Remove HybridSource from ValuesSource to avoid
CI failure. (#3237)
---
.../flink/cdc/connectors/values/source/ValuesDataSource.java | 7 +--
.../cdc/connectors/values/source/ValuesDataSourceHelper.java | 8 +++-
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git
a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSource.java
b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSource.java
index 88a723fce..61699a5d0 100644
---
a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSource.java
+++
b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSource.java
@@ -35,7 +35,6 @@ import org.apache.flink.cdc.common.source.EventSourceProvider;
import org.apache.flink.cdc.common.source.FlinkSourceProvider;
import org.apache.flink.cdc.common.source.MetadataAccessor;
import org.apache.flink.cdc.connectors.values.ValuesDatabase;
-import org.apache.flink.connector.base.source.hybrid.HybridSource;
import org.apache.flink.core.io.InputStatus;
import org.apache.flink.core.io.SimpleVersionedSerializer;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
@@ -73,11 +72,7 @@ public class ValuesDataSource implements DataSource {
@Override
public EventSourceProvider getEventSourceProvider() {
ValuesDataSourceHelper.setSourceEvents(eventSetId);
-HybridSource hybridSource =
-HybridSource.builder(new ValuesSource(failAtPos, eventSetId,
true))
-.addSource(new ValuesSource(failAtPos, eventSetId,
false))
-.build();
-return FlinkSourceProvider.of(hybridSource);
+return FlinkSourceProvider.of(new ValuesSource(failAtPos, eventSetId,
false));
}
@Override
diff --git
a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSourceHelper.java
b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSourceHelper.java
index 431cbaf17..24ba78165 100644
---
a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSourceHelper.java
+++
b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSourceHelper.java
@@ -73,7 +73,13 @@ public class ValuesDataSourceHelper {
// use default enum of SINGLE_SPLIT_SINGLE_TABLE
sourceEvents = singleSplitSingleTable();
}
-return sourceEvents;
+// put all events into one list to avoid CI failure and make sure that
SchemaChangeEvent are
+// sent in order.
+List mergeEvents = new ArrayList<>();
+for (List events : sourceEvents) {
+mergeEvents.addAll(events);
+}
+return Collections.singletonList(mergeEvents);
}
/** set sourceEvents using custom events. */
(flink-cdc) branch master updated: [FLINK-34903][cdc][mysql] Add mysql-pipeline-connector with tables.exclude option to exclude unnecessary tables (#3186)
This is an automated email from the ASF dual-hosted git repository.
renqs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
The following commit(s) were added to refs/heads/master by this push:
new 0d797a602 [FLINK-34903][cdc][mysql] Add mysql-pipeline-connector with
tables.exclude option to exclude unnecessary tables (#3186)
0d797a602 is described below
commit 0d797a6021ea2f3973f8dc74c0c741b695e85a12
Author: Thorne <46524102+shiy...@users.noreply.github.com>
AuthorDate: Fri Apr 19 15:49:32 2024 +0800
[FLINK-34903][cdc][mysql] Add mysql-pipeline-connector with tables.exclude
option to exclude unnecessary tables (#3186)
---
docs/content.zh/docs/connectors/mysql.md | 8
docs/content/docs/connectors/mysql.md | 8
.../mysql/factory/MySqlDataSourceFactory.java | 27 ++---
.../mysql/source/MySqlDataSourceOptions.java | 12 ++
.../mysql/source/MySqlDataSourceFactoryTest.java | 45 ++
5 files changed, 95 insertions(+), 5 deletions(-)
diff --git a/docs/content.zh/docs/connectors/mysql.md
b/docs/content.zh/docs/connectors/mysql.md
index 564dd3dbf..3410cdea0 100644
--- a/docs/content.zh/docs/connectors/mysql.md
+++ b/docs/content.zh/docs/connectors/mysql.md
@@ -107,6 +107,14 @@ pipeline:
需要注意的是，点号（.）被视为数据库和表名的分隔符。
如果需要在正则表达式中使用点（.）来匹配任何字符，必须使用反斜杠对点进行转义。
例如，db0.\.*, db1.user_table_[0-9]+, db[1-2].[app|web]order_\.*
+
+ tables.exclude
+ optional
+ (none)
+ String
+ 需要排除的 MySQL 数据库的表名，参数会在tables参数后发生排除作用。表名支持正则表达式，以排除满足正则表达式的多个表。
+ 用法和tables参数相同
+
schema-change.enabled
optional
diff --git a/docs/content/docs/connectors/mysql.md
b/docs/content/docs/connectors/mysql.md
index c6ef38252..879920fec 100644
--- a/docs/content/docs/connectors/mysql.md
+++ b/docs/content/docs/connectors/mysql.md
@@ -109,6 +109,14 @@ pipeline:
If there is a need to use a dot (.) in a regular expression to match
any character, it is necessary to escape the dot with a backslash.
eg. db0.\.*, db1.user_table_[0-9]+, db[1-2].[app|web]order_\.*
+
+ tables.exclude
+ optional
+ (none)
+ String
+ Table name of the MySQL database to exclude, parameter will have an
exclusion effect after the tables parameter. The table-name also supports
regular expressions to exclude multiple tables that satisfy the regular
expressions.
+ The usage is the same as the tables parameter
+
schema-change.enabled
optional
diff --git
a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/factory/MySqlDataSourceFactory.java
b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/factory/MySqlDataSourceFactory.java
index 744041d25..dc9972fe0 100644
---
a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/factory/MySqlDataSourceFactory.java
+++
b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/factory/MySqlDataSourceFactory.java
@@ -42,9 +42,11 @@ import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.time.ZoneId;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Collectors;
import static
org.apache.flink.cdc.connectors.mysql.source.MySqlDataSourceOptions.CHUNK_KEY_EVEN_DISTRIBUTION_FACTOR_LOWER_BOUND;
import static
org.apache.flink.cdc.connectors.mysql.source.MySqlDataSourceOptions.CHUNK_KEY_EVEN_DISTRIBUTION_FACTOR_UPPER_BOUND;
@@ -70,6 +72,7 @@ import static
org.apache.flink.cdc.connectors.mysql.source.MySqlDataSourceOption
import static
org.apache.flink.cdc.connectors.mysql.source.MySqlDataSourceOptions.SERVER_ID;
import static
org.apache.flink.cdc.connectors.mysql.source.MySqlDataSourceOptions.SERVER_TIME_ZONE;
import static
org.apache.flink.cdc.connectors.mysql.source.MySqlDataSourceOptions.TABLES;
+import static
org.apache.flink.cdc.connectors.mysql.source.MySqlDataSourceOptions.TABLES_EXCLUDE;
import static
org.apache.flink.cdc.connectors.mysql.source.MySqlDataSourceOptions.USERNAME;
import static
org.apache.flink.cdc.connectors.mysql.source.utils.ObjectUtils.doubleCompare;
import static
org.apache.flink.cdc.debezium.table.DebeziumOptions.getDebeziumProperties;
@@ -93,6 +96,7 @@ public class MySqlDataSourceFactory implements
DataSourceFactory {
String username = config.get(USERNAME);
String password = config.get(PASSWORD);
String tables = config.get(TABLES);
+String tablesExclude = config.get(TABLES_EXCLUDE);
String serverId = validat
(flink) branch master updated (a312a3bdd25 -> 131358b918b)
This is an automated email from the ASF dual-hosted git repository. chesnay pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from a312a3bdd25 [FLINK-35045][state] Support ByteBufferReadable for HadoopDataInputStream add 10c84df8c56 [hotfix] Delete pointless test add 131358b918b [FLINK-35159] Transition ExecutionGraph to RUNNING after slot assignment No new revisions were added by this update. Summary of changes: .../scheduler/adaptive/AdaptiveScheduler.java | 1 - .../scheduler/adaptive/CreatingExecutionGraph.java | 5 +- .../scheduler/adaptive/AdaptiveSchedulerTest.java | 55 -- .../adaptive/CreatingExecutionGraphTest.java | 8 +++- 4 files changed, 10 insertions(+), 59 deletions(-)
Re: [I] flink-sql-connector-oracle-cdc-2.4.0采集出现异常 [flink-cdc]
Aiden-Rose commented on issue #2234: URL: https://github.com/apache/flink-cdc/issues/2234#issuecomment-2066176302 我遇到了同样的问题，后面版本有解决的吗 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@flink.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
(flink-connector-aws) branch main updated: [FLINK-35115][Connectors/Kinesis] Allow kinesis consumer to snapshotState after cancelling operator
This is an automated email from the ASF dual-hosted git repository.
dannycranmer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-aws.git
The following commit(s) were added to refs/heads/main by this push:
new 8d29147 [FLINK-35115][Connectors/Kinesis] Allow kinesis consumer to
snapshotState after cancelling operator
8d29147 is described below
commit 8d29147b9e6c0a7d27399662c6023ad634363764
Author: Aleksandr Pilipenko
AuthorDate: Wed Apr 17 17:05:23 2024 +0100
[FLINK-35115][Connectors/Kinesis] Allow kinesis consumer to snapshotState
after cancelling operator
---
.../connectors/kinesis/FlinkKinesisConsumer.java | 13 +-
.../kinesis/FlinkKinesisConsumerTest.java | 304 +++--
2 files changed, 179 insertions(+), 138 deletions(-)
diff --git
a/flink-connector-aws/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
b/flink-connector-aws/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
index c229a1c..06d0acc 100644
---
a/flink-connector-aws/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
+++
b/flink-connector-aws/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
@@ -148,7 +148,17 @@ public class FlinkKinesisConsumer extends
RichParallelSourceFunction
private transient HashMap
sequenceNumsToRestore;
+/**
+ * Flag used to control reading from Kinesis: source will read data while
value is true. Changed
+ * to false after {@link #cancel()} has been called.
+ */
private volatile boolean running = true;
+/**
+ * Flag identifying that operator had been closed. True only after {@link
#close()} has been
+ * called. Used to control behaviour of snapshotState: state can be
persisted after operator has
+ * been cancelled (during stop-with-savepoint workflow), but not after
operator has been closed.
+ */
+private volatile boolean closed = false;
//
// State for Checkpoint
@@ -419,6 +429,7 @@ public class FlinkKinesisConsumer extends
RichParallelSourceFunction
@Override
public void close() throws Exception {
cancel();
+closed = true;
// safe-guard when the fetcher has been interrupted, make sure to not
leak resources
// application might be stopped before connector subtask has been
started
// so we must check if the fetcher is actually created
@@ -478,7 +489,7 @@ public class FlinkKinesisConsumer extends
RichParallelSourceFunction
@Override
public void snapshotState(FunctionSnapshotContext context) throws
Exception {
-if (!running) {
+if (closed) {
LOG.debug("snapshotState() called on closed source; returning
null.");
} else {
if (LOG.isDebugEnabled()) {
diff --git
a/flink-connector-aws/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
b/flink-connector-aws/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
index f8cd5ab..82836df 100644
---
a/flink-connector-aws/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
+++
b/flink-connector-aws/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
@@ -39,6 +39,7 @@ import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import
org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
import org.apache.flink.streaming.api.operators.StreamSource;
+import
org.apache.flink.streaming.api.operators.collect.utils.MockFunctionSnapshotContext;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.api.windowing.time.Time;
import
org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants;
@@ -71,7 +72,6 @@ import
com.amazonaws.services.kinesis.model.SequenceNumberRange;
import com.amazonaws.services.kinesis.model.Shard;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.mockito.Matchers;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
@@ -97,10 +97,13 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.never;
import
svn commit: r68649 - in /dev/flink/flink-connector-aws-4.3.0-rc2: ./ flink-connector-aws-4.3.0-src.tgz flink-connector-aws-4.3.0-src.tgz.asc flink-connector-aws-4.3.0-src.tgz.sha512
Author: dannycranmer Date: Fri Apr 19 09:51:08 2024 New Revision: 68649 Log: Add flink-connector-aws-4.3.0-rc2 Added: dev/flink/flink-connector-aws-4.3.0-rc2/ dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz (with props) dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz.asc dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz.sha512 Added: dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz == Binary file - no diff available. Propchange: dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz -- svn:mime-type = application/octet-stream Added: dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz.asc == --- dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz.asc (added) +++ dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz.asc Fri Apr 19 09:51:08 2024 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCAAdFiEED3nyr7I1G8KWeFRFkfnB7BJf2NsFAmYiPoYACgkQkfnB7BJf +2NsemQ/+LUfar2B32/5DibL8FTSDIceAE//oNXxbTtzmOYGJL+kgiavbNBz4eqJC +xKRP4CXmNs74yFcrHIkNL9Qls54A1H7L0+y0xmwQEubP3t7gQx90wHfdyfiBEmKB +cB1PLZjoOD/NlzFILZRKE+dVGgfQoaKW3OxQaxBExzLpXlOUtOLMQ424Htuw4s/V +LJaT/5uHg3XeBYpIVvmq6KkbAGE4WyOa8O+aE1UvvlV/haFvdTMN3NdL459fCjAA +tO6LY3z6FXG+AQY5rU//3/P9rsvbxfUQQbY4M8tVVK8K7zeXAPuLXH5WnCIiUDmR +id+uYBg2Pyud5RnOp9gKxVnf+zEJ+GP/w6OWq2LTiU+W8d18Ayvtl69sQl4g/GjN +mbAxSeZ+ht/g1rN/kyMT08T6khczmjWHlmqyKuv+Yj+tjKpCg26X40gGnqAFtSuD +BXHfz8TYgty1SX6I9wLUQY0EWV4jx3NjRtq7oRuiPMT9LZd+M0PGhYDkXigiD+B+ +LJm3XWEJZ1SQ7AlTyCeYG+C2aNuN0dWI3uyksBbsHqw80XW04ywsNa3xXS3wOF1s +LRZKihAhxINwAF8A/5VJE1m/7tU5kNt51jm+8TREA8KmMg2/VDO8dKfUhhg0JL/f +xijYOPNFHuRh7BUD1JdOPupBve1gq5lSLutO5A9xjeeToOURCzc= +=KPb8 +-END PGP SIGNATURE- Added: dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz.sha512 == --- dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz.sha512 (added) +++ dev/flink/flink-connector-aws-4.3.0-rc2/flink-connector-aws-4.3.0-src.tgz.sha512 Fri Apr 19 09:51:08 2024 @@ -0,0 +1 @@ +dcf27e76b1aab0c2e6c6aad18ae5e8753925c20e4a9186fbaed73d1d745a4900d4861bf7f5ebf0c1e1186d18c6bbc05c757022a590fe9db5d9d44cee3c192285 flink-connector-aws-4.3.0-src.tgz
(flink-connector-aws) annotated tag v4.3.0-rc2 updated (a68f076 -> 009e133)
This is an automated email from the ASF dual-hosted git repository. dannycranmer pushed a change to annotated tag v4.3.0-rc2 in repository https://gitbox.apache.org/repos/asf/flink-connector-aws.git *** WARNING: tag v4.3.0-rc2 was modified! *** from a68f076 (commit) to 009e133 (tag) tagging a68f076d3b94309e7a38dc1829c9bde97e3709d9 (commit) by Danny Cranmer on Fri Apr 19 10:50:48 2024 +0100 - Log - v4.3.0-rc2 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEED3nyr7I1G8KWeFRFkfnB7BJf2NsFAmYiPngACgkQkfnB7BJf 2Nv9tw/+P/HMw+wZ8Af7zXDUv0I+ql/8DPQmXntwpDOqEhC1b3ACr9HR7KbFw0Ab /AWYJH2IGOxm/3EmgJ9L75egOm05b0gKE1GfEqNsqMO2qjTFDGUC0bHflsyzKZnh 9fNpGbRTFkVnm5szylF1CSLCiXHrpFqw0zH++ypt76YPVnvovSHDoM2oW3TRfiTu B2JZyyECs99kirBLl2MbQRc4YrPzPPhFaUuAu95L7yUU+1q8jdFIuuX8iLrpSDmu ZXz1Z3LejnAGlUiWJr6bZVUYzkKefqc2vYK1eedK2bYbKY6W0LoEb4ht81jzxftA bQwZuJDyk2ldPoB708UtgajKUpYRZ1ikQq2udINAQiUuhR8h8Osv9COYk+0s3T+s LTkIXpu/ybx9VbAjEuwEJk/l3uekURGXfAxFTLSWlFsw9tLPMrWnU0hvdUU2EuAe 3BUpfbf3cA2ZZ1qrUnyGylJKVTU2J/SHFtYawenrrB0Mb2xA84A9zMaRuQA3EPt2 I/dFHnfweqaUHGrXGHeT5hpUIoLWUX5xcHo1RuybzFgRhRR7nyKToSgENtgy2qqV UT9Qk50/IhCNbYBp9Qjm30CScvVzd6t4kRjJsvKr4TdVd7ig9BnACj43gt3e/MBq dsfZYaC/PwyT47MbevyXgpg1mG0n/1aD1q7JTaYuSQD7ZIzRx/M= =vdcp -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
(flink) branch master updated (131358b918b -> 39779829b88)
This is an automated email from the ASF dual-hosted git repository. dannycranmer pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 131358b918b [FLINK-35159] Transition ExecutionGraph to RUNNING after slot assignment add 39779829b88 [FLINK-34954][core] Kryo Input bug fix No new revisions were added by this update. Summary of changes: .../flink/api/java/typeutils/runtime/NoFetchingInput.java | 11 ++- 1 file changed, 2 insertions(+), 9 deletions(-)
(flink-connector-rabbitmq) branch dependabot/maven/org.apache.commons-commons-compress-1.26.0 deleted (was 107b33c)
This is an automated email from the ASF dual-hosted git repository. dannycranmer pushed a change to branch dependabot/maven/org.apache.commons-commons-compress-1.26.0 in repository https://gitbox.apache.org/repos/asf/flink-connector-rabbitmq.git was 107b33c Bump org.apache.commons:commons-compress from 1.24.0 to 1.26.0 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(flink-connector-rabbitmq) branch main updated: [hotfix] Bump version to 3.1-SNAPSHOT
This is an automated email from the ASF dual-hosted git repository. dannycranmer pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/flink-connector-rabbitmq.git The following commit(s) were added to refs/heads/main by this push: new 1b0e2a8 [hotfix] Bump version to 3.1-SNAPSHOT 1b0e2a8 is described below commit 1b0e2a8a1e43be38d4fafead0c465ceb68bcc67b Author: Danny Cranmer AuthorDate: Fri Apr 19 11:10:09 2024 +0100 [hotfix] Bump version to 3.1-SNAPSHOT --- flink-connector-rabbitmq/pom.xml | 2 +- flink-sql-connector-rabbitmq/pom.xml | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flink-connector-rabbitmq/pom.xml b/flink-connector-rabbitmq/pom.xml index fc6d527..93cb8de 100644 --- a/flink-connector-rabbitmq/pom.xml +++ b/flink-connector-rabbitmq/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connector-rabbitmq-parent - 3.0-SNAPSHOT + 3.1-SNAPSHOT flink-connector-rabbitmq diff --git a/flink-sql-connector-rabbitmq/pom.xml b/flink-sql-connector-rabbitmq/pom.xml index ca479f0..68c5779 100644 --- a/flink-sql-connector-rabbitmq/pom.xml +++ b/flink-sql-connector-rabbitmq/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connector-rabbitmq-parent - 3.0-SNAPSHOT + 3.1-SNAPSHOT flink-sql-connector-rabbitmq diff --git a/pom.xml b/pom.xml index 211db42..43381e5 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ under the License. org.apache.flink flink-connector-rabbitmq-parent -3.0-SNAPSHOT +3.1-SNAPSHOT Flink : Connectors : RabbitMQ : Parent pom 2022
(flink-connector-rabbitmq) branch main updated: [FLINK-35142][Connectors/RabbitMQ] Drop support for Flink 1.17
This is an automated email from the ASF dual-hosted git repository.
dannycranmer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-rabbitmq.git
The following commit(s) were added to refs/heads/main by this push:
new 8e8a022 [FLINK-35142][Connectors/RabbitMQ] Drop support for Flink 1.17
8e8a022 is described below
commit 8e8a02282e248fc73f20f1ac1fe21250629e9c8f
Author: Danny Cranmer
AuthorDate: Fri Apr 19 11:02:41 2024 +0100
[FLINK-35142][Connectors/RabbitMQ] Drop support for Flink 1.17
---
.github/workflows/push_pr.yml | 6 ++
.github/workflows/weekly.yml | 12
2 files changed, 2 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/push_pr.yml b/.github/workflows/push_pr.yml
index f4289dc..da8a577 100644
--- a/.github/workflows/push_pr.yml
+++ b/.github/workflows/push_pr.yml
@@ -28,11 +28,9 @@ jobs:
compile_and_test:
strategy:
matrix:
-flink: [ 1.16-SNAPSHOT, 1.17-SNAPSHOT ]
-jdk: [ '8, 11' ]
+flink: [ 1.18-SNAPSHOT ]
+jdk: [ '8, 11, 17' ]
include:
- - flink: 1.18-SNAPSHOT
-jdk: '8, 11, 17'
- flink: 1.19-SNAPSHOT
jdk: '8, 11, 17, 21'
uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml
index 4e8372b..8385763 100644
--- a/.github/workflows/weekly.yml
+++ b/.github/workflows/weekly.yml
@@ -30,12 +30,6 @@ jobs:
strategy:
matrix:
flink_branches: [{
- flink: 1.16-SNAPSHOT,
- branch: main
-}, {
- flink: 1.17-SNAPSHOT,
- branch: main
-}, {
flink: 1.18-SNAPSHOT,
jdk: '8, 11, 17',
branch: main
@@ -43,12 +37,6 @@ jobs:
flink: 1.19-SNAPSHOT,
jdk: '8, 11, 17, 21',
branch: main
-}, {
- flink: 1.16.3,
- branch: v3.0
-}, {
- flink: 1.17.2,
- branch: v3.0
}, {
flink: 1.18.0,
jdk: '8, 11, 17',
(flink-connector-rabbitmq) 01/02: [hotfix] Update copyright year to 2024
This is an automated email from the ASF dual-hosted git repository. dannycranmer pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/flink-connector-rabbitmq.git commit ddbfa8f9d66d6a2c20c678382d52b43d977360cf Author: Hang Ruan AuthorDate: Mon Jan 29 13:53:37 2024 +0800 [hotfix] Update copyright year to 2024 --- flink-sql-connector-rabbitmq/src/main/resources/META-INF/NOTICE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-sql-connector-rabbitmq/src/main/resources/META-INF/NOTICE b/flink-sql-connector-rabbitmq/src/main/resources/META-INF/NOTICE index e4f475f..9fbfdc5 100644 --- a/flink-sql-connector-rabbitmq/src/main/resources/META-INF/NOTICE +++ b/flink-sql-connector-rabbitmq/src/main/resources/META-INF/NOTICE @@ -1,5 +1,5 @@ flink-sql-connector-rabbitmq -Copyright 2014-2023 The Apache Software Foundation +Copyright 2014-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/).
(flink-connector-rabbitmq) branch main updated (1b0e2a8 -> 6b8a906)
This is an automated email from the ASF dual-hosted git repository. dannycranmer pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/flink-connector-rabbitmq.git from 1b0e2a8 [hotfix] Bump version to 3.1-SNAPSHOT new ddbfa8f [hotfix] Update copyright year to 2024 new 6b8a906 [hotfix] Fix the dependency convergence rule The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: flink-connector-rabbitmq/pom.xml | 6 ++ .../src/main/resources/META-INF/NOTICE | 2 +- pom.xml| 14 +- 3 files changed, 20 insertions(+), 2 deletions(-)
(flink-connector-rabbitmq) 02/02: [hotfix] Fix the dependency convergence rule
This is an automated email from the ASF dual-hosted git repository. dannycranmer pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/flink-connector-rabbitmq.git commit 6b8a9060629e72fa08c9f78507c26ea5f27c2f3f Author: Hang Ruan AuthorDate: Tue Feb 6 17:15:03 2024 +0800 [hotfix] Fix the dependency convergence rule --- flink-connector-rabbitmq/pom.xml | 6 ++ pom.xml | 14 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/flink-connector-rabbitmq/pom.xml b/flink-connector-rabbitmq/pom.xml index 93cb8de..22d0c3a 100644 --- a/flink-connector-rabbitmq/pom.xml +++ b/flink-connector-rabbitmq/pom.xml @@ -80,6 +80,12 @@ under the License. org.testcontainers rabbitmq test + + + com.fasterxml.jackson.core + jackson-annotations + + diff --git a/pom.xml b/pom.xml index 43381e5..91701fe 100644 --- a/pom.xml +++ b/pom.xml @@ -110,6 +110,12 @@ under the License. org.testcontainers junit-jupiter test + + +com.fasterxml.jackson.core +jackson-annotations + + @@ -136,6 +142,12 @@ under the License. org.apache.flink flink-test-utils-junit test + + +com.fasterxml.jackson.core +jackson-annotations + + @@ -400,4 +412,4 @@ under the License. - \ No newline at end of file +
(flink-connector-rabbitmq) branch main updated: [FLINK-35174][Connectors/RabbitMQ] Bump commons-compress to 1.26.1
This is an automated email from the ASF dual-hosted git repository. dannycranmer pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/flink-connector-rabbitmq.git The following commit(s) were added to refs/heads/main by this push: new 66e323a [FLINK-35174][Connectors/RabbitMQ] Bump commons-compress to 1.26.1 66e323a is described below commit 66e323a3e79befc08ae03f2789a8aa94b343d504 Author: Danny Cranmer AuthorDate: Fri Apr 19 11:14:21 2024 +0100 [FLINK-35174][Connectors/RabbitMQ] Bump commons-compress to 1.26.1 --- pom.xml | 16 +++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 91701fe..23c5fd7 100644 --- a/pom.xml +++ b/pom.xml @@ -320,7 +320,21 @@ under the License. org.apache.commons commons-compress -1.24.0 +1.26.1 + + + + +org.apache.commons +commons-lang3 +3.14.0 + + + + +commons-io +commons-io +2.15.1
Re: [I] The XML dependency of Flink Oracle CDC conflicts with the XML related classes of Java8：ERROR StatusLogger Caught javax.xml.parsers.ParserConfigurationException [flink-cdc]
Aiden-Rose commented on issue #3195: URL: https://github.com/apache/flink-cdc/issues/3195#issuecomment-2066353680 I also encountered the same problem. Have you resolved it -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@flink.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
(flink) branch release-1.18 updated: [FLINK-35159] Transition ExecutionGraph to RUNNING after slot assignment
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch release-1.18
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.18 by this push:
new aacc735806a [FLINK-35159] Transition ExecutionGraph to RUNNING after
slot assignment
aacc735806a is described below
commit aacc735806acf1d63fa732706e079bc2ca1bb4fc
Author: Chesnay Schepler
AuthorDate: Thu Apr 18 19:10:42 2024 +0200
[FLINK-35159] Transition ExecutionGraph to RUNNING after slot assignment
---
.../flink/runtime/scheduler/adaptive/AdaptiveScheduler.java | 1 -
.../flink/runtime/scheduler/adaptive/CreatingExecutionGraph.java | 5 -
.../runtime/scheduler/adaptive/CreatingExecutionGraphTest.java| 8 ++--
3 files changed, 10 insertions(+), 4 deletions(-)
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/AdaptiveScheduler.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/AdaptiveScheduler.java
index 4ee22c95848..34539d23e04 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/AdaptiveScheduler.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/AdaptiveScheduler.java
@@ -1096,7 +1096,6 @@ public class AdaptiveScheduler
executionGraphWithVertexParallelism.getExecutionGraph();
executionGraph.start(componentMainThreadExecutor);
-executionGraph.transitionToRunning();
executionGraph.setInternalTaskFailuresListener(
new UpdateSchedulerNgOnInternalFailuresListener(this));
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraph.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraph.java
index c876fe6ad1d..21055945372 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraph.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraph.java
@@ -124,7 +124,6 @@ public class CreatingExecutionGraph implements State {
operatorCoordinatorHandlerFactory.create(executionGraph, context);
operatorCoordinatorHandler.initializeOperatorCoordinators(
context.getMainThreadExecutor());
-operatorCoordinatorHandler.startAllOperatorCoordinators();
final String updatedPlan =
JsonPlanGenerator.generatePlan(
executionGraph.getJobID(),
@@ -138,6 +137,10 @@ public class CreatingExecutionGraph implements State {
.iterator(),
executionGraphWithVertexParallelism.getVertexParallelism());
executionGraph.setJsonPlan(updatedPlan);
+
+executionGraph.transitionToRunning();
+operatorCoordinatorHandler.startAllOperatorCoordinators();
+
context.goToExecuting(
result.getExecutionGraph(),
executionGraphHandler,
diff --git
a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraphTest.java
b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraphTest.java
index 2375a194206..69e5f589b19 100644
---
a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraphTest.java
+++
b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraphTest.java
@@ -157,8 +157,12 @@ public class CreatingExecutionGraphTest extends TestLogger
{
ignored ->
CreatingExecutionGraph.AssignmentResult.notPossible());
context.setExpectWaitingForResources();
-executionGraphWithVertexParallelismFuture.complete(
-getGraph(new StateTrackingMockExecutionGraph()));
+final StateTrackingMockExecutionGraph executionGraph =
+new StateTrackingMockExecutionGraph();
+
+
executionGraphWithVertexParallelismFuture.complete(getGraph(executionGraph));
+
+
assertThat(executionGraph.getState()).isEqualTo(JobStatus.INITIALIZING);
}
}
(flink) branch release-1.19 updated: [FLINK-35159] Transition ExecutionGraph to RUNNING after slot assignment
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.19 by this push:
new 87ed9ccc210 [FLINK-35159] Transition ExecutionGraph to RUNNING after
slot assignment
87ed9ccc210 is described below
commit 87ed9ccc2103457ba91f6ca45adfd2bfcc75c9ac
Author: Chesnay Schepler
AuthorDate: Thu Apr 18 19:10:42 2024 +0200
[FLINK-35159] Transition ExecutionGraph to RUNNING after slot assignment
---
.../flink/runtime/scheduler/adaptive/AdaptiveScheduler.java | 1 -
.../flink/runtime/scheduler/adaptive/CreatingExecutionGraph.java | 5 -
.../runtime/scheduler/adaptive/CreatingExecutionGraphTest.java| 8 ++--
3 files changed, 10 insertions(+), 4 deletions(-)
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/AdaptiveScheduler.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/AdaptiveScheduler.java
index 5f6438ce181..238c594fd55 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/AdaptiveScheduler.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/AdaptiveScheduler.java
@@ -1187,7 +1187,6 @@ public class AdaptiveScheduler
executionGraphWithVertexParallelism.getExecutionGraph();
executionGraph.start(componentMainThreadExecutor);
-executionGraph.transitionToRunning();
executionGraph.setInternalTaskFailuresListener(
new UpdateSchedulerNgOnInternalFailuresListener(this));
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraph.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraph.java
index da90ef1468d..e9b1317e46e 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraph.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraph.java
@@ -123,7 +123,6 @@ public class CreatingExecutionGraph extends
StateWithoutExecutionGraph {
operatorCoordinatorHandlerFactory.create(executionGraph, context);
operatorCoordinatorHandler.initializeOperatorCoordinators(
context.getMainThreadExecutor());
-operatorCoordinatorHandler.startAllOperatorCoordinators();
final String updatedPlan =
JsonPlanGenerator.generatePlan(
executionGraph.getJobID(),
@@ -137,6 +136,10 @@ public class CreatingExecutionGraph extends
StateWithoutExecutionGraph {
.iterator(),
executionGraphWithVertexParallelism.getVertexParallelism());
executionGraph.setJsonPlan(updatedPlan);
+
+executionGraph.transitionToRunning();
+operatorCoordinatorHandler.startAllOperatorCoordinators();
+
context.goToExecuting(
result.getExecutionGraph(),
executionGraphHandler,
diff --git
a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraphTest.java
b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraphTest.java
index b831b3bb62f..0f89cdf7e12 100644
---
a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraphTest.java
+++
b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/CreatingExecutionGraphTest.java
@@ -93,8 +93,12 @@ class CreatingExecutionGraphTest {
ignored ->
CreatingExecutionGraph.AssignmentResult.notPossible());
context.setExpectWaitingForResources();
-executionGraphWithVertexParallelismFuture.complete(
-getGraph(new StateTrackingMockExecutionGraph()));
+final StateTrackingMockExecutionGraph executionGraph =
+new StateTrackingMockExecutionGraph();
+
+
executionGraphWithVertexParallelismFuture.complete(getGraph(executionGraph));
+
+
assertThat(executionGraph.getState()).isEqualTo(JobStatus.INITIALIZING);
}
@Test
(flink-kubernetes-operator) branch main updated: [FLINK-34574] Add CPU and memory size autoscaler quota
This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new baad9008 [FLINK-34574] Add CPU and memory size autoscaler quota
baad9008 is described below
commit baad90088ea5b5b240186a530a79b64fb84cc77e
Author: Gabor Somogyi
AuthorDate: Fri Apr 19 17:36:24 2024 +0200
[FLINK-34574] Add CPU and memory size autoscaler quota
---
.../generated/auto_scaler_configuration.html | 12 +
.../apache/flink/autoscaler/ScalingExecutor.java | 113 +-
.../flink/autoscaler/ScalingMetricCollector.java | 15 +-
.../flink/autoscaler/config/AutoScalerOptions.java | 17 +
.../flink/autoscaler/topology/JobTopology.java | 18 +-
.../flink/autoscaler/topology/VertexInfo.java | 23 +-
.../runtime/rest/messages/job/JobDetailsInfo.java | 448 +
.../json/SlotSharingGroupIDDeserializer.java | 44 ++
.../json/SlotSharingGroupIDSerializer.java | 43 ++
.../messages/json/SlotSharingGroupIdConverter.java | 42 ++
.../MetricsCollectionAndEvaluationTest.java| 11 +-
.../flink/autoscaler/ScalingExecutorTest.java | 193 -
.../autoscaler/ScalingMetricCollectorTest.java | 10 +
.../flink/autoscaler/TestingAutoscalerUtils.java | 9 +-
.../flink/autoscaler/topology/JobTopologyTest.java | 2 +-
.../operator/config/FlinkConfigBuilder.java| 18 +-
.../kubernetes/operator/utils/EventRecorder.java | 1 +
.../operator/config/FlinkConfigBuilderTest.java| 4 +
18 files changed, 977 insertions(+), 46 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/auto_scaler_configuration.html
b/docs/layouts/shortcodes/generated/auto_scaler_configuration.html
index b3466b63..d9bed101 100644
--- a/docs/layouts/shortcodes/generated/auto_scaler_configuration.html
+++ b/docs/layouts/shortcodes/generated/auto_scaler_configuration.html
@@ -116,6 +116,18 @@
Double
Percentage threshold for switching to observed from busy time
based true processing rate if the measurement is off by at least the configured
fraction. For example 0.15 means we switch to observed if the busy time based
computation is at least 15% higher during catchup.
+
+job.autoscaler.quota.cpu
+(none)
+Double
+Quota of the CPU count. When scaling would go beyond this
number the the scaling is not going to happen.
+
+
+job.autoscaler.quota.memory
+(none)
+MemorySize
+Quota of the memory size. When scaling would go beyond this
number the the scaling is not going to happen.
+
job.autoscaler.restart.time
5 min
diff --git
a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingExecutor.java
b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingExecutor.java
index 71efc160..af325371 100644
---
a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingExecutor.java
+++
b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingExecutor.java
@@ -33,6 +33,7 @@ import org.apache.flink.autoscaler.utils.ResourceCheckUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.MemorySize;
import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.runtime.instance.SlotSharingGroupId;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.slf4j.Logger;
@@ -67,6 +68,9 @@ public class ScalingExecutor> {
public static final String HEAP_USAGE_MESSAGE =
"Heap Usage %s is above the allowed limit for scaling operations.
Please adjust the available memory manually.";
+public static final String RESOURCE_QUOTA_REACHED_MESSAGE =
+"Resource usage is above the allowed limit for scaling operations.
Please adjust the resource quota manually.";
+
private static final Logger LOG =
LoggerFactory.getLogger(ScalingExecutor.class);
private final JobVertexScaler jobVertexScaler;
@@ -129,8 +133,10 @@ public class ScalingExecutor> {
scalingSummaries,
autoScalerEventHandler);
-if (scalingWouldExceedClusterResources(
-configOverrides.newConfigWithOverrides(conf),
+var memoryTuningEnabled =
conf.get(AutoScalerOptions.MEMORY_TUNING_ENABLED);
+if (scalingWouldExceedMaxResources(
+memoryTuningEnabled ?
configOverrides.newConfigWithOverrides(conf) : conf,
+jobTopology,
evaluatedMetrics,
scalingSummaries,
context)) {
@@ -280,6 +286,30 @@ public class ScalingExecutor> {
return false;
}
+@VisibleForTesting
+protected boolean scalingWouldExceedMaxResourc
(flink) branch master updated (39779829b88 -> 7c4dec63eb4)
This is an automated email from the ASF dual-hosted git repository.
thw pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 39779829b88 [FLINK-34954][core] Kryo Input bug fix
add 7c4dec63eb4 [FLINK-28048][connectors] Introduce Source API alternative
to FiniteTestSource (#23777)
No new revisions were added by this update.
Summary of changes:
.../f7a4e6fa-e7de-48c9-a61e-c13e83f0c72e | 118 +++
flink-connectors/flink-connector-datagen/pom.xml | 16 +++
...tion.java => IndexLookupGeneratorFunction.java} | 145 ---
.../datagen/source/DataGeneratorSource.java| 2 +-
...ittingSourceReaderWithCheckpointsInBetween.java | 158 +
.../datagen/source/TestDataGenerators.java | 83 +++
flink-connectors/flink-connector-hive/pom.xml | 8 ++
.../flink/connectors/hive/HiveTableSinkITCase.java | 49 ---
.../connectors/hive/HiveTableSourceITCase.java | 40 +++---
.../source/lib/util/IteratorSourceReaderBase.java | 10 +-
flink-formats/flink-avro/pom.xml | 8 ++
.../formats/avro/AvroStreamingFileSinkITCase.java | 25 +++-
flink-formats/flink-compress/pom.xml | 8 ++
.../formats/compress/CompressionFactoryITCase.java | 10 +-
flink-formats/flink-csv/pom.xml| 8 ++
flink-formats/flink-hadoop-bulk/pom.xml| 8 ++
.../bulk/HadoopPathBasedPartFileWriterITCase.java | 15 +-
flink-formats/flink-json/pom.xml | 8 ++
flink-formats/flink-orc/pom.xml| 8 ++
.../flink/orc/writer/OrcBulkWriterITCase.java | 10 +-
flink-formats/flink-parquet/pom.xml| 8 ++
.../avro/AvroParquetStreamingFileSinkITCase.java | 21 ++-
.../ParquetProtoStreamingFileSinkITCase.java | 10 +-
flink-formats/flink-sequence-file/pom.xml | 8 ++
.../SequenceStreamingFileSinkITCase.java | 12 +-
flink-table/flink-table-planner/pom.xml| 10 +-
.../runtime/stream/sql/CompactionITCaseBase.java | 20 +--
flink-tests/pom.xml| 2 +
.../flink/test/streaming/runtime/SinkITCase.java | 75 +-
29 files changed, 652 insertions(+), 251 deletions(-)
copy
flink-connectors/flink-connector-datagen/src/main/java/org/apache/flink/connector/datagen/functions/{FromElementsGeneratorFunction.java
=> IndexLookupGeneratorFunction.java} (61%)
create mode 100644
flink-connectors/flink-connector-datagen/src/main/java/org/apache/flink/connector/datagen/source/DoubleEmittingSourceReaderWithCheckpointsInBetween.java
create mode 100644
flink-connectors/flink-connector-datagen/src/test/java/org/apache/flink/connector/datagen/source/TestDataGenerators.java
