tillrohrmann commented on a change in pull request #16946:
URL: https://github.com/apache/flink/pull/16946#discussion_r701202114
##########
File path:
flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
##########
@@ -331,6 +331,22 @@
+ "JobManager could be faster, since no
reverse DNS lookup is performed. "
+ "However, local input split assignment
(such as for HDFS files) may be impacted.");
+ @Documentation.Section(Documentation.Sections.EXPERT_JOB_MANAGER)
+ public static final ConfigOption<Integer> JOB_MANAGER_FUTURE_THREADS =
+ key("jobmanager.future-executor.numThreads")
+ .intType()
+ .noDefaultValue()
+ .withDescription(
+ "The number of jobmanager future-executor threads
for shared services.");
+
+ @Documentation.Section(Documentation.Sections.EXPERT_JOB_MANAGER)
+ public static final ConfigOption<Integer> JOB_MANAGER_IO_THREADS =
+ key("jobmanager.io-executor.numThreads")
+ .intType()
+ .noDefaultValue()
+ .withDescription(
+ "The number of jobmanager io-executor threads for
shared services.");
Review comment:
Same here.
##########
File path:
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServices.java
##########
@@ -147,10 +163,19 @@ public static JobManagerSharedServices fromConfiguration(
failOnJvmMetaspaceOomError ? fatalErrorHandler
: null,
checkClassLoaderLeak));
+ final int numJobManagerFutureThreads =
+ config.getInteger(
+ JobManagerOptions.JOB_MANAGER_FUTURE_THREADS,
Hardware.getNumberCPUCores());
final ScheduledExecutorService futureExecutor =
Executors.newScheduledThreadPool(
- Hardware.getNumberCPUCores(),
- new ExecutorThreadFactory("jobmanager-future"));
+ numJobManagerFutureThreads, new
ExecutorThreadFactory("jobmanager-future"));
+
+ final int numJobManagerIoThreads =
+ config.getInteger(
+ JobManagerOptions.JOB_MANAGER_IO_THREADS,
Hardware.getNumberCPUCores());
Review comment:
nit: `Hardware.getNumberCPUCores` could be deduplicated.
##########
File path:
flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServicesTest.java
##########
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.runtime.blob.BlobServer;
+import org.apache.flink.runtime.blob.VoidBlobStore;
+import org.apache.flink.runtime.util.Hardware;
+import org.apache.flink.runtime.util.TestingFatalErrorHandler;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests auxiliary shared services created by {@link JobManagerSharedServices}
and used by the
+ * {@link JobMaster}.
+ */
+public class JobManagerSharedServicesTest extends TestLogger {
+
+ private static final int NUM_FUTURE_THREADS = 8;
+ private static final int NUM_IO_THREADS = 4;
+ private static final int CPU_CORES = Hardware.getNumberCPUCores();
+
+ @Test
+ public void testFutureExecutorNoConfiguration() throws Exception {
+ Configuration config = new Configuration();
+
+ final JobManagerSharedServices jobManagerSharedServices =
+ JobManagerSharedServices.fromConfiguration(
+ config,
+ new BlobServer(config, new VoidBlobStore()),
+ new TestingFatalErrorHandler());
+
+ CountDownLatch releasedLatch = new CountDownLatch(CPU_CORES);
+ CountDownLatch unreleasedLatch = new CountDownLatch(CPU_CORES + 1);
+ ScheduledExecutorService futureExecutor =
jobManagerSharedServices.getFutureExecutor();
+ Runnable countsDown =
+ () -> {
+ releasedLatch.countDown();
+ unreleasedLatch.countDown();
+ };
+ for (int i = 0; i < CPU_CORES; i++) {
+ futureExecutor.execute(countsDown);
+ }
+ // Give the executor enough time to process all the Runnables
+ releasedLatch.await(1L, TimeUnit.SECONDS);
+ assertEquals(0, releasedLatch.getCount());
+ assertEquals(1, unreleasedLatch.getCount());
+ jobManagerSharedServices.shutdown();
+ }
+
+ @Test
+ public void testFutureExecutorConfiguration() throws Exception {
+ Configuration config = new Configuration();
+ config.setInteger(JobManagerOptions.JOB_MANAGER_FUTURE_THREADS,
NUM_FUTURE_THREADS);
+
+ final JobManagerSharedServices jobManagerSharedServices =
+ JobManagerSharedServices.fromConfiguration(
+ config,
+ new BlobServer(config, new VoidBlobStore()),
+ new TestingFatalErrorHandler());
+
+ CountDownLatch releasedLatch = new CountDownLatch(NUM_FUTURE_THREADS);
+ CountDownLatch unreleasedLatch = new CountDownLatch(NUM_FUTURE_THREADS
+ 1);
+ ScheduledExecutorService futureExecutor =
jobManagerSharedServices.getFutureExecutor();
+ Runnable countsDown =
+ () -> {
+ releasedLatch.countDown();
+ unreleasedLatch.countDown();
+ };
Review comment:
I think the the runnable also needs to block. Otherwise the test will
also pass if the `futureExecutor` has only a single thread.
##########
File path:
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServices.java
##########
@@ -147,10 +163,19 @@ public static JobManagerSharedServices fromConfiguration(
failOnJvmMetaspaceOomError ? fatalErrorHandler
: null,
checkClassLoaderLeak));
+ final int numJobManagerFutureThreads =
+ config.getInteger(
+ JobManagerOptions.JOB_MANAGER_FUTURE_THREADS,
Hardware.getNumberCPUCores());
final ScheduledExecutorService futureExecutor =
Executors.newScheduledThreadPool(
- Hardware.getNumberCPUCores(),
- new ExecutorThreadFactory("jobmanager-future"));
+ numJobManagerFutureThreads, new
ExecutorThreadFactory("jobmanager-future"));
+
+ final int numJobManagerIoThreads =
+ config.getInteger(
+ JobManagerOptions.JOB_MANAGER_IO_THREADS,
Hardware.getNumberCPUCores());
+ final ScheduledExecutorService ioExecutor =
+ Executors.newScheduledThreadPool(
Review comment:
Could probably use `newFixedThreadPool`
##########
File path:
flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
##########
@@ -331,6 +331,22 @@
+ "JobManager could be faster, since no
reverse DNS lookup is performed. "
+ "However, local input split assignment
(such as for HDFS files) may be impacted.");
+ @Documentation.Section(Documentation.Sections.EXPERT_JOB_MANAGER)
+ public static final ConfigOption<Integer> JOB_MANAGER_FUTURE_THREADS =
+ key("jobmanager.future-executor.numThreads")
+ .intType()
+ .noDefaultValue()
+ .withDescription(
+ "The number of jobmanager future-executor threads
for shared services.");
Review comment:
I think we should add that this defaults to the number of CPUs if no
value has been specified.
##########
File path:
flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServicesTest.java
##########
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.runtime.blob.BlobServer;
+import org.apache.flink.runtime.blob.VoidBlobStore;
+import org.apache.flink.runtime.util.Hardware;
+import org.apache.flink.runtime.util.TestingFatalErrorHandler;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests auxiliary shared services created by {@link JobManagerSharedServices}
and used by the
+ * {@link JobMaster}.
+ */
+public class JobManagerSharedServicesTest extends TestLogger {
+
+ private static final int NUM_FUTURE_THREADS = 8;
+ private static final int NUM_IO_THREADS = 4;
+ private static final int CPU_CORES = Hardware.getNumberCPUCores();
+
+ @Test
+ public void testFutureExecutorNoConfiguration() throws Exception {
+ Configuration config = new Configuration();
+
+ final JobManagerSharedServices jobManagerSharedServices =
+ JobManagerSharedServices.fromConfiguration(
+ config,
+ new BlobServer(config, new VoidBlobStore()),
+ new TestingFatalErrorHandler());
+
+ CountDownLatch releasedLatch = new CountDownLatch(CPU_CORES);
+ CountDownLatch unreleasedLatch = new CountDownLatch(CPU_CORES + 1);
+ ScheduledExecutorService futureExecutor =
jobManagerSharedServices.getFutureExecutor();
+ Runnable countsDown =
+ () -> {
+ releasedLatch.countDown();
+ unreleasedLatch.countDown();
+ };
+ for (int i = 0; i < CPU_CORES; i++) {
+ futureExecutor.execute(countsDown);
+ }
+ // Give the executor enough time to process all the Runnables
+ releasedLatch.await(1L, TimeUnit.SECONDS);
+ assertEquals(0, releasedLatch.getCount());
+ assertEquals(1, unreleasedLatch.getCount());
+ jobManagerSharedServices.shutdown();
+ }
+
+ @Test
+ public void testFutureExecutorConfiguration() throws Exception {
+ Configuration config = new Configuration();
+ config.setInteger(JobManagerOptions.JOB_MANAGER_FUTURE_THREADS,
NUM_FUTURE_THREADS);
+
+ final JobManagerSharedServices jobManagerSharedServices =
+ JobManagerSharedServices.fromConfiguration(
+ config,
+ new BlobServer(config, new VoidBlobStore()),
+ new TestingFatalErrorHandler());
+
+ CountDownLatch releasedLatch = new CountDownLatch(NUM_FUTURE_THREADS);
+ CountDownLatch unreleasedLatch = new CountDownLatch(NUM_FUTURE_THREADS
+ 1);
+ ScheduledExecutorService futureExecutor =
jobManagerSharedServices.getFutureExecutor();
+ Runnable countsDown =
+ () -> {
+ releasedLatch.countDown();
+ unreleasedLatch.countDown();
+ };
+ for (int i = 0; i < NUM_FUTURE_THREADS; i++) {
+ futureExecutor.execute(countsDown);
+ }
+ // Give the executor enough time to process all the Runnables
+ releasedLatch.await(1L, TimeUnit.SECONDS);
+ assertEquals(0, releasedLatch.getCount());
+ assertEquals(1, unreleasedLatch.getCount());
+ jobManagerSharedServices.shutdown();
+ }
+
+ @Test
+ public void testIoExecutorNoConfiguration() throws Exception {
+ Configuration config = new Configuration();
+
+ final JobManagerSharedServices jobManagerSharedServices =
+ JobManagerSharedServices.fromConfiguration(
+ config,
+ new BlobServer(config, new VoidBlobStore()),
+ new TestingFatalErrorHandler());
+
+ CountDownLatch releasedLatch = new CountDownLatch(CPU_CORES);
+ CountDownLatch unreleasedLatch = new CountDownLatch(CPU_CORES + 1);
+ ExecutorService ioExecutor = jobManagerSharedServices.getIoExecutor();
+ Runnable countsDown =
+ () -> {
+ releasedLatch.countDown();
+ unreleasedLatch.countDown();
+ };
+ for (int i = 0; i < CPU_CORES; i++) {
+ ioExecutor.execute(countsDown);
+ }
+ // Give the executor enough time to process all the Runnables
+ releasedLatch.await(1L, TimeUnit.SECONDS);
+ assertEquals(0, releasedLatch.getCount());
+ assertEquals(1, unreleasedLatch.getCount());
+ jobManagerSharedServices.shutdown();
+ }
+
+ @Test
+ public void testIoExecutorConfiguration() throws Exception {
+ Configuration config = new Configuration();
+ config.setInteger(JobManagerOptions.JOB_MANAGER_IO_THREADS,
NUM_IO_THREADS);
+
+ final JobManagerSharedServices jobManagerSharedServices =
+ JobManagerSharedServices.fromConfiguration(
+ config,
+ new BlobServer(config, new VoidBlobStore()),
+ new TestingFatalErrorHandler());
+
+ CountDownLatch releasedLatch = new CountDownLatch(NUM_IO_THREADS);
+ CountDownLatch unreleasedLatch = new CountDownLatch(NUM_IO_THREADS +
1);
+ ExecutorService ioExecutor = jobManagerSharedServices.getIoExecutor();
+ Runnable countsDown =
+ () -> {
+ releasedLatch.countDown();
+ unreleasedLatch.countDown();
+ };
+ for (int i = 0; i < NUM_IO_THREADS; i++) {
+ ioExecutor.execute(countsDown);
+ }
+ // Give the executor enough time to process all the Runnables
+ releasedLatch.await(1L, TimeUnit.SECONDS);
+ assertEquals(0, releasedLatch.getCount());
+ assertEquals(1, unreleasedLatch.getCount());
+ jobManagerSharedServices.shutdown();
+ }
Review comment:
Same for these two tests.
##########
File path:
flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
##########
@@ -148,7 +149,9 @@
private final HeartbeatServices heartbeatServices;
- private final ScheduledExecutorService scheduledExecutorService;
+ private final ScheduledExecutorService futureExecutor;
+
+ private final ExecutorService ioExecutor;
Review comment:
Maybe `Executor` is enough here.
##########
File path:
flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServicesTest.java
##########
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.runtime.blob.BlobServer;
+import org.apache.flink.runtime.blob.VoidBlobStore;
+import org.apache.flink.runtime.util.Hardware;
+import org.apache.flink.runtime.util.TestingFatalErrorHandler;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests auxiliary shared services created by {@link JobManagerSharedServices}
and used by the
+ * {@link JobMaster}.
+ */
+public class JobManagerSharedServicesTest extends TestLogger {
+
+ private static final int NUM_FUTURE_THREADS = 8;
+ private static final int NUM_IO_THREADS = 4;
+ private static final int CPU_CORES = Hardware.getNumberCPUCores();
+
+ @Test
+ public void testFutureExecutorNoConfiguration() throws Exception {
+ Configuration config = new Configuration();
+
+ final JobManagerSharedServices jobManagerSharedServices =
+ JobManagerSharedServices.fromConfiguration(
+ config,
+ new BlobServer(config, new VoidBlobStore()),
+ new TestingFatalErrorHandler());
+
+ CountDownLatch releasedLatch = new CountDownLatch(CPU_CORES);
+ CountDownLatch unreleasedLatch = new CountDownLatch(CPU_CORES + 1);
+ ScheduledExecutorService futureExecutor =
jobManagerSharedServices.getFutureExecutor();
+ Runnable countsDown =
+ () -> {
+ releasedLatch.countDown();
+ unreleasedLatch.countDown();
+ };
+ for (int i = 0; i < CPU_CORES; i++) {
+ futureExecutor.execute(countsDown);
+ }
+ // Give the executor enough time to process all the Runnables
+ releasedLatch.await(1L, TimeUnit.SECONDS);
+ assertEquals(0, releasedLatch.getCount());
+ assertEquals(1, unreleasedLatch.getCount());
+ jobManagerSharedServices.shutdown();
+ }
+
+ @Test
+ public void testFutureExecutorConfiguration() throws Exception {
+ Configuration config = new Configuration();
+ config.setInteger(JobManagerOptions.JOB_MANAGER_FUTURE_THREADS,
NUM_FUTURE_THREADS);
+
+ final JobManagerSharedServices jobManagerSharedServices =
+ JobManagerSharedServices.fromConfiguration(
+ config,
+ new BlobServer(config, new VoidBlobStore()),
+ new TestingFatalErrorHandler());
+
+ CountDownLatch releasedLatch = new CountDownLatch(NUM_FUTURE_THREADS);
+ CountDownLatch unreleasedLatch = new CountDownLatch(NUM_FUTURE_THREADS
+ 1);
+ ScheduledExecutorService futureExecutor =
jobManagerSharedServices.getFutureExecutor();
+ Runnable countsDown =
+ () -> {
+ releasedLatch.countDown();
+ unreleasedLatch.countDown();
+ };
+ for (int i = 0; i < NUM_FUTURE_THREADS; i++) {
+ futureExecutor.execute(countsDown);
+ }
+ // Give the executor enough time to process all the Runnables
+ releasedLatch.await(1L, TimeUnit.SECONDS);
+ assertEquals(0, releasedLatch.getCount());
+ assertEquals(1, unreleasedLatch.getCount());
+ jobManagerSharedServices.shutdown();
Review comment:
We probably can deduplicate this part and the part in the
`testFutureExecutorNoConfiguration`.
##########
File path:
flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
##########
@@ -331,6 +331,22 @@
+ "JobManager could be faster, since no
reverse DNS lookup is performed. "
+ "However, local input split assignment
(such as for HDFS files) may be impacted.");
+ @Documentation.Section(Documentation.Sections.EXPERT_JOB_MANAGER)
+ public static final ConfigOption<Integer> JOB_MANAGER_FUTURE_THREADS =
+ key("jobmanager.future-executor.numThreads")
+ .intType()
+ .noDefaultValue()
+ .withDescription(
+ "The number of jobmanager future-executor threads
for shared services.");
+
+ @Documentation.Section(Documentation.Sections.EXPERT_JOB_MANAGER)
+ public static final ConfigOption<Integer> JOB_MANAGER_IO_THREADS =
+ key("jobmanager.io-executor.numThreads")
+ .intType()
+ .noDefaultValue()
+ .withDescription(
+ "The number of jobmanager io-executor threads for
shared services.");
+
Review comment:
I think that we also need to rebuild the documentation via `mvn package
-Dgenerate-config-docs -pl flink-docs`. Otherwise the configuration options
won't appear in the documentation and the documentation completeness test will
fail.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@flink.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
