[flink] branch master updated: [FLINK-31781][runtime] Introduces contender ID in LeaderElectionService interface
This is an automated email from the ASF dual-hosted git repository.
mapohl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 62b792d58c4 [FLINK-31781][runtime] Introduces contender ID in
LeaderElectionService interface
62b792d58c4 is described below
commit 62b792d58c4f8d5b494b50daad2e5fc5047af330
Author: Matthias Pohl
AuthorDate: Tue May 16 18:42:49 2023 +0200
[FLINK-31781][runtime] Introduces contender ID in LeaderElectionService
interface
Signed-off-by: Matthias Pohl
---
.../highavailability/AbstractHaServices.java | 7 +-
.../AbstractLeaderElectionService.java | 36 +++---
.../leaderelection/DefaultLeaderElection.java | 23 ++--
.../DefaultLeaderElectionService.java | 60 ++---
.../leaderelection/LeaderElectionService.java | 5 +-
.../JobMasterServiceLeadershipRunnerTest.java | 4 +-
.../DefaultLeaderElectionServiceTest.java | 105 +++-
.../leaderelection/DefaultLeaderElectionTest.java | 140 -
.../runtime/leaderelection/LeaderElectionTest.java | 2 +-
...KeeperLeaderElectionConnectionHandlingTest.java | 3 +-
.../ZooKeeperLeaderElectionTest.java | 9 +-
11 files changed, 247 insertions(+), 147 deletions(-)
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/AbstractHaServices.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/AbstractHaServices.java
index 4c24110c42a..401e63cc959 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/AbstractHaServices.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/AbstractHaServices.java
@@ -247,7 +247,12 @@ public abstract class AbstractHaServices implements
HighAvailabilityServices {
leaderElectionService.startLeaderElectionBackend();
closeableRegistry.registerCloseable(leaderElectionService);
-return leaderElectionService.createLeaderElection();
+// the leaderName which is passed as a contenderID here is not
actively used within the
+// DefaultLeaderElectionService for now - this will change in a future
step where the
+// DefaultLeaderElectionService will start to use
MultipleComponentLeaderElectionDriver
+// instead of LeaderElectionDriver and fully replace
+// DefaultMultipleComponentLeaderElectionService (FLINK-31783)
+return
leaderElectionService.createLeaderElection("unused-contender-id");
}
/**
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/AbstractLeaderElectionService.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/AbstractLeaderElectionService.java
index 0abad419329..eef2d731fa7 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/AbstractLeaderElectionService.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/AbstractLeaderElectionService.java
@@ -27,29 +27,37 @@ import java.util.UUID;
public abstract class AbstractLeaderElectionService implements
LeaderElectionService {
@Override
-public LeaderElection createLeaderElection() {
-return new DefaultLeaderElection(this);
+public LeaderElection createLeaderElection(String contenderID) {
+return new DefaultLeaderElection(this, contenderID);
}
/**
- * Registers the given {@link LeaderContender} with the underlying {@code
- * LeaderElectionService}. Leadership changes are starting to be reported
to the {@code
+ * Registers the {@link LeaderContender} under the {@code contenderID}
with the underlying
+ * {@code LeaderElectionService}. Leadership changes are starting to be
reported to the {@code
* LeaderContender}.
*/
-protected abstract void register(LeaderContender contender) throws
Exception;
+protected abstract void register(String contenderID, LeaderContender
contender)
+throws Exception;
-/** Removes the passed {@code LeaderContender} from the {@code
LeaderElectionService}. */
-protected abstract void remove(LeaderContender contender);
+/**
+ * Removes the {@code LeaderContender} from the {@code
LeaderElectionService} that is associated
+ * with the {@code contenderID}.
+ */
+protected abstract void remove(String contenderID);
-/** Confirms the leadership with the given session ID and address. */
-protected abstract void confirmLeadership(UUID leaderSessionID, String
leaderAddress);
+/**
+ * Confirms the leadership with the {@code leaderSessionID} and {@code
leaderAddress} for the
+ * {@link LeaderContender} that is associated with the {@code contenderID}.
+ */
+protected abstract void confirmLeadership(
+String contenderID, UUID leaderSession
[flink] branch master updated (62b792d58c4 -> 83ebe45c895)
This is an automated email from the ASF dual-hosted git repository. chesnay pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 62b792d58c4 [FLINK-31781][runtime] Introduces contender ID in LeaderElectionService interface add 83ebe45c895 [FLINK-25000][build] Java 17 uses Scala 2.12.15 No new revisions were added by this update. Summary of changes: flink-scala/pom.xml | 18 ++ flink-streaming-scala/pom.xml | 21 + .../migration/ScalaSerializersMigrationTest.scala | 3 +++ pom.xml | 5 + 4 files changed, 47 insertions(+)
[flink] branch master updated (83ebe45c895 -> d8547fafd99)
This is an automated email from the ASF dual-hosted git repository. chesnay pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 83ebe45c895 [FLINK-25000][build] Java 17 uses Scala 2.12.15 add d8547fafd99 [FLINK-32358][rpc] Add rpc system loading priority No new revisions were added by this update. Summary of changes: .../runtime/rpc/akka/AkkaRpcSystemLoader.java | 7 ++ .../rpc/akka/FallbackAkkaRpcSystemLoader.java | 5 + .../org/apache/flink/runtime/rpc/RpcSystem.java| 9 +- .../apache/flink/runtime/rpc/RpcSystemLoader.java | 6 ++ .../apache/flink/runtime/rpc/RpcSystemTest.java| 115 + 5 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 flink-rpc/flink-rpc-core/src/test/java/org/apache/flink/runtime/rpc/RpcSystemTest.java
[flink] branch master updated (d8547fafd99 -> 201456c5501)
This is an automated email from the ASF dual-hosted git repository. mapohl pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from d8547fafd99 [FLINK-32358][rpc] Add rpc system loading priority add 201456c5501 [FLINK-32177][runtime] Refactors MultipleComponentLeaderElectionDriver.Listener.notifyAllKnownLeaderInformation signature (#22642) No new revisions were added by this update. Summary of changes: ...netesMultipleComponentLeaderElectionDriver.java | 20 +++ ...sMultipleComponentLeaderElectionDriverTest.java | 19 --- ...aultMultipleComponentLeaderElectionService.java | 27 +++-- .../leaderelection/LeaderInformationRegister.java | 51 + .../LeaderInformationWithComponentId.java | 65 -- .../MultipleComponentLeaderElectionDriver.java | 11 +--- ...MultipleComponentLeaderElectionServiceTest.java | 21 +++ .../leaderelection/LeaderElectionEvent.java| 14 ++--- .../TestingLeaderElectionListener.java | 7 +-- ...rMultipleComponentLeaderElectionDriverTest.java | 3 +- 10 files changed, 94 insertions(+), 144 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/LeaderInformationRegister.java delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/LeaderInformationWithComponentId.java
[flink] branch master updated (201456c5501 -> ed854c0240c)
This is an automated email from the ASF dual-hosted git repository. mapohl pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 201456c5501 [FLINK-32177][runtime] Refactors MultipleComponentLeaderElectionDriver.Listener.notifyAllKnownLeaderInformation signature (#22642) add ed854c0240c [FLINK-32368][test] Fixes wrong interface implementation in KubernetesTestFixture No new revisions were added by this update. Summary of changes: .../KubernetesCheckpointIDCounterTest.java | 4 ++-- .../KubernetesLeaderElectionDriverTest.java| 27 +++--- .../KubernetesStateHandleStoreTest.java| 6 ++--- .../highavailability/KubernetesTestFixture.java| 5 ++-- 4 files changed, 21 insertions(+), 21 deletions(-)
[flink] branch master updated (ed854c0240c -> 2940c02c986)
This is an automated email from the ASF dual-hosted git repository. chesnay pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from ed854c0240c [FLINK-32368][test] Fixes wrong interface implementation in KubernetesTestFixture add 89c15245531 [hotfix] Fix license indentation add 1b6db829502 [hotfix][yarn][tests] Simplify string construction add 1297e18a4f7 [hotfix][yarn] Set -XX:+IgnoreUnrecognizedVMOptions for task managers add e323a895712 [hotfix][build] Set -XX:+IgnoreUnrecognizedVMOptions in surefire VMs add 2940c02c986 [FLINK-25002][build] Add java 17 add-opens/add-exports JVM arguments No new revisions were added by this update. Summary of changes: .mvn/jvm.config| 6 + flink-clients/pom.xml | 7 + flink-connectors/flink-connector-base/pom.xml | 7 + flink-connectors/flink-connector-files/pom.xml | 7 + flink-connectors/flink-connector-kafka/pom.xml | 6 + flink-core/pom.xml | 18 ++ flink-dist/src/main/resources/flink-conf.yaml | 5 +- .../flink-queryable-state-test/pom.xml | 3 + .../test_local_recovery_and_scheduling.sh | 2 + flink-examples/flink-examples-table/pom.xml| 7 + flink-filesystems/flink-hadoop-fs/pom.xml | 11 + flink-formats/flink-avro/pom.xml | 13 ++ flink-formats/flink-csv/pom.xml| 7 + flink-formats/flink-json/pom.xml | 7 + flink-formats/flink-orc/pom.xml| 9 + flink-formats/flink-parquet/pom.xml| 7 + flink-fs-tests/pom.xml | 7 + flink-java/pom.xml | 9 + flink-kubernetes/pom.xml | 4 + flink-libraries/flink-cep/pom.xml | 7 + flink-libraries/flink-state-processing-api/pom.xml | 9 + flink-metrics/flink-metrics-jmx/pom.xml| 7 + flink-python/pom.xml | 9 + flink-python/pyflink/pyflink_gateway_server.py | 6 +- .../flink-queryable-state-runtime/pom.xml | 7 + flink-runtime/pom.xml | 14 ++ .../runtime/clusterframework/BootstrapTools.java | 5 + .../clusterframework/BootstrapToolsTest.java | 232 - .../flink/runtime/testutils/TestJvmProcess.java| 5 + flink-scala/pom.xml| 7 + .../flink-statebackend-changelog/pom.xml | 7 + .../flink-statebackend-rocksdb/pom.xml | 7 + flink-streaming-java/pom.xml | 9 + flink-streaming-scala/pom.xml | 7 + flink-table/flink-sql-client/pom.xml | 9 + flink-table/flink-sql-gateway/pom.xml | 9 + flink-table/flink-sql-jdbc-driver/pom.xml | 7 + flink-table/flink-table-planner/pom.xml| 17 ++ flink-table/flink-table-runtime/pom.xml| 9 + .../apache/flink/test/util/TestProcessBuilder.java | 7 + flink-tests/pom.xml| 9 + flink-yarn-tests/pom.xml | 8 + flink-yarn/pom.xml | 4 + pom.xml| 10 +- 44 files changed, 510 insertions(+), 59 deletions(-)
[flink] 01/02: [FLINK-32327][ci] Java 17 skips python tests
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit d11266ee10230c4602e2abd9d79eea40a870adb3
Author: Chesnay Schepler
AuthorDate: Thu Jun 15 16:23:21 2023 +0200
[FLINK-32327][ci] Java 17 skips python tests
---
tools/azure-pipelines/jobs-template.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tools/azure-pipelines/jobs-template.yml
b/tools/azure-pipelines/jobs-template.yml
index 8a586de96c8..3e6a14b3fc8 100644
--- a/tools/azure-pipelines/jobs-template.yml
+++ b/tools/azure-pipelines/jobs-template.yml
@@ -149,6 +149,7 @@ jobs:
# Test
- script: ${{parameters.environment}}
./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh
$(module)
+condition: or(not(eq(${{parameters.jdk}}, '17')), not(eq(variables.module,
'python')))
displayName: Test - $(module)
env:
IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET)
@@ -162,7 +163,7 @@ jobs:
# upload debug artifacts
- task: PublishPipelineArtifact@1
-condition: not(eq('$(DEBUG_FILES_OUTPUT_DIR)', ''))
+condition: and(not(eq('$(DEBUG_FILES_OUTPUT_DIR)', '')),
or(not(eq(${{parameters.jdk}}, '17')), not(eq(variables.module, 'python'
displayName: Upload Logs
inputs:
targetPath: $(DEBUG_FILES_OUTPUT_DIR)
[flink] branch master updated (2940c02c986 -> 4624cc47bec)
This is an automated email from the ASF dual-hosted git repository. chesnay pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 2940c02c986 [FLINK-25002][build] Add java 17 add-opens/add-exports JVM arguments new d11266ee102 [FLINK-32327][ci] Java 17 skips python tests new 4624cc47bec [FLINK-32369][ci] Setup Java 17 cron build The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: tools/azure-pipelines/build-apache-repo.yml | 11 +++ tools/azure-pipelines/jobs-template.yml | 3 ++- tools/ci/compile.sh | 6 -- 3 files changed, 17 insertions(+), 3 deletions(-)
[flink] 02/02: [FLINK-32369][ci] Setup Java 17 cron build
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 4624cc47bec135f369bcdf159b68bdd4566ce5af
Author: Chesnay Schepler
AuthorDate: Fri Jun 16 15:05:30 2023 +0200
[FLINK-32369][ci] Setup Java 17 cron build
---
tools/azure-pipelines/build-apache-repo.yml | 11 +++
tools/ci/compile.sh | 6 --
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/tools/azure-pipelines/build-apache-repo.yml
b/tools/azure-pipelines/build-apache-repo.yml
index eea833ad826..8a63b17a5de 100644
--- a/tools/azure-pipelines/build-apache-repo.yml
+++ b/tools/azure-pipelines/build-apache-repo.yml
@@ -138,6 +138,17 @@ stages:
run_end_to_end: true
container: flink-build-container
jdk: 11
+ - template: jobs-template.yml
+parameters:
+ stage_name: cron_jdk17
+ test_pool_definition:
+name: Default
+ e2e_pool_definition:
+vmImage: 'ubuntu-20.04'
+ environment: PROFILE="-Dflink.hadoop.version=2.10.2 -Dscala-2.12
-Djdk11 -Djdk17"
+ run_end_to_end: true
+ container: flink-build-container
+ jdk: 17
- template: jobs-template.yml
parameters:
stage_name: cron_adaptive_scheduler
diff --git a/tools/ci/compile.sh b/tools/ci/compile.sh
index c2c68be7a47..4c38931085b 100755
--- a/tools/ci/compile.sh
+++ b/tools/ci/compile.sh
@@ -110,8 +110,10 @@ EXIT_CODE=$(($EXIT_CODE+$?))
echo " Run license check "
find $MVN_VALIDATION_DIR
-
-${CI_DIR}/license_check.sh $MVN_CLEAN_COMPILE_OUT $CI_DIR $(pwd)
$MVN_VALIDATION_DIR || exit $?
+# We use a different Scala version with Java 17
+if [[ ${PROFILE} != *"jdk17"* ]]; then
+ ${CI_DIR}/license_check.sh $MVN_CLEAN_COMPILE_OUT $CI_DIR $(pwd)
$MVN_VALIDATION_DIR || exit $?
+fi
exit $EXIT_CODE
[flink-connector-kafka] branch main updated (27e2bd30 -> 910fc5a3)
This is an automated email from the ASF dual-hosted git repository. tzulitai pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/flink-connector-kafka.git from 27e2bd30 [FLINK-32289][docs] Fix incorrect metadata column type in examples add 910fc5a3 [FLINK-31747] Remove Confluent Avro formats from externalized Kafka connector repo No new revisions were added by this update. Summary of changes: .idea/vcs.xml | 1 + .../connectors/table/formats/avro-confluent.md | 294 -- .../docs/connectors/table/formats/debezium.md | 455 --- .../connectors/table/formats/avro-confluent.md | 301 -- .../docs/connectors/table/formats/debezium.md | 454 --- flink-confluent-schema-registry-e2e-tests/pom.xml | 135 - .../registry/test/TestAvroConsumerConfluent.java | 112 .../src/main/resources/avro/user.avsc | 27 - .../flink-end-to-end-tests-common-kafka/pom.xml| 4 +- flink-connector-kafka/pom.xml | 2 +- .../kafka/table/KafkaDynamicTableFactoryTest.java | 2 +- .../1483f895-db24-4888-a2fa-991d602eaafc | 0 .../bfcdf96e-7dcd-4715-9f39-73483823ec16 | 0 .../archunit-violations/stored.rules | 4 - .../flink-avro-confluent-registry/pom.xml | 204 --- .../confluent/AvroConfluentFormatOptions.java | 131 - .../confluent/CachedSchemaCoderProvider.java | 84 --- ...ConfluentRegistryAvroDeserializationSchema.java | 205 --- .../ConfluentRegistryAvroSerializationSchema.java | 143 - .../confluent/ConfluentSchemaRegistryCoder.java| 93 --- .../confluent/RegistryAvroFormatFactory.java | 264 - .../DebeziumAvroDeserializationSchema.java | 207 --- .../debezium/DebeziumAvroFormatFactory.java| 198 --- .../debezium/DebeziumAvroSerializationSchema.java | 151 - .../org.apache.flink.table.factories.Factory | 17 - .../architecture/TestCodeArchitectureTest.java | 40 -- .../confluent/CachedSchemaCoderProviderTest.java | 143 - .../ConfluentSchemaRegistryCoderTest.java | 77 --- .../confluent/RegistryAvroFormatFactoryTest.java | 294 -- .../RegistryAvroRowDataSeDeSchemaTest.java | 194 --- .../debezium/DebeziumAvroFormatFactoryTest.java| 249 .../debezium/DebeziumAvroSerDeSchemaTest.java | 242 .../org.junit.jupiter.api.extension.Extension | 16 - .../src/test/resources/archunit.properties | 31 - .../src/test/resources/debezium-avro-delete.avro | Bin 147 -> 0 bytes .../src/test/resources/debezium-avro-insert.avro | Bin 143 -> 0 bytes .../src/test/resources/debezium-avro-update.avro | Bin 188 -> 0 bytes .../src/test/resources/debezium-test-schema.json | 191 -- .../src/test/resources/test-keystore.jks | Bin 2327 -> 0 bytes .../flink-sql-avro-confluent-registry/pom.xml | 120 .../src/main/resources/META-INF/NOTICE | 32 -- .../licenses/LICENSE.jakarta-annotation-api| 637 - .../META-INF/licenses/LICENSE.jakarta-inject | 637 - .../META-INF/licenses/LICENSE.jakarta-ws-rs-api| 637 - .../licenses/LICENSE.osgi-resource-locator | 637 - .../resources/META-INF/licenses/LICENSE.zstd-jni | 26 - flink-formats-kafka/pom.xml| 41 -- pom.xml| 2 - 48 files changed, 5 insertions(+), 7729 deletions(-) delete mode 100644 docs/content.zh/docs/connectors/table/formats/avro-confluent.md delete mode 100644 docs/content.zh/docs/connectors/table/formats/debezium.md delete mode 100644 docs/content/docs/connectors/table/formats/avro-confluent.md delete mode 100644 docs/content/docs/connectors/table/formats/debezium.md delete mode 100644 flink-confluent-schema-registry-e2e-tests/pom.xml delete mode 100644 flink-confluent-schema-registry-e2e-tests/src/main/java/org/apache/flink/schema/registry/test/TestAvroConsumerConfluent.java delete mode 100644 flink-confluent-schema-registry-e2e-tests/src/main/resources/avro/user.avsc delete mode 100644 flink-formats-kafka/flink-avro-confluent-registry/archunit-violations/1483f895-db24-4888-a2fa-991d602eaafc delete mode 100644 flink-formats-kafka/flink-avro-confluent-registry/archunit-violations/bfcdf96e-7dcd-4715-9f39-73483823ec16 delete mode 100644 flink-formats-kafka/flink-avro-confluent-registry/archunit-violations/stored.rules delete mode 100644 flink-formats-kafka/flink-avro-confluent-registry/pom.xml delete mode 100644 flink-formats-kafka/flink-avro-confluent-registry/src/main/java/org/apache/flink/formats/avro/registry/confluent/AvroConfluentFormatOptions.java delete mode 100644 flink-formats-kafka/flink-avro-confluent-registry/src/main/java/org/apache/flink/formats
