Repository: flink Updated Branches: refs/heads/master fc0001c85 -> 3c8a673f9
[FLINK-8458][config][docs] Add config of credit-based network buffers This closes #5317. Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3c8a673f Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3c8a673f Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3c8a673f Branch: refs/heads/master Commit: 3c8a673f99dc6ba28987b827cfc65f348a26ed32 Parents: fc0001c Author: Zhijiang <wangzhijiang...@aliyun.com> Authored: Fri Feb 23 11:29:16 2018 +0800 Committer: Stefan Richter <s.rich...@data-artisans.com> Committed: Mon Mar 5 12:29:28 2018 +0100 ---------------------------------------------------------------------- docs/_includes/generated/netty_configuration.html | 15 +++++++++++++++ .../flink/configuration/TaskManagerOptions.java | 17 +++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/3c8a673f/docs/_includes/generated/netty_configuration.html ---------------------------------------------------------------------- diff --git a/docs/_includes/generated/netty_configuration.html b/docs/_includes/generated/netty_configuration.html index 47c48c0..e97fda4 100644 --- a/docs/_includes/generated/netty_configuration.html +++ b/docs/_includes/generated/netty_configuration.html @@ -42,5 +42,20 @@ <td style="word-wrap: break-word;">"nio"</td> <td>The Netty transport type, either "nio" or "epoll"</td> </tr> + <tr> + <td><h5>taskmanager.network.credit-based-flow-control.enabled</h5></td> + <td style="word-wrap: break-word;">true</td> + <td>Boolean flag to enable/disable network credit-based flow control</td> + </tr> + <tr> + <td><h5>taskmanager.network.memory.buffers-per-channel</h5></td> + <td style="word-wrap: break-word;">2</td> + <td>Number of network buffers to use for each outgoing/incoming channel (subpartition/input channel). In credit-based flow control mode, this indicates how many credits are exclusive in each input channel. It should be configured at least 2 for good performance. 1 buffer is for receiving in-flight data in the subpartition and 1 buffer is for parallel serialization</td> + </tr> + <tr> + <td><h5>taskmanager.network.memory.floating-buffers-per-gate</h5></td> + <td style="word-wrap: break-word;">8</td> + <td>Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels. The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be increased in case of higher round trip times between nodes and/or larger number of machines in the cluster</td> + </tr> </tbody> </table> http://git-wip-us.apache.org/repos/asf/flink/blob/3c8a673f/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java ---------------------------------------------------------------------- diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java index cc3284c..4e08fda 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java @@ -269,7 +269,10 @@ public class TaskManagerOptions { public static final ConfigOption<Integer> NETWORK_BUFFERS_PER_CHANNEL = key("taskmanager.network.memory.buffers-per-channel") .defaultValue(2) - .withDescription("Number of network buffers to use for each outgoing/incoming channel (subpartition/input channel)."); + .withDescription("Number of network buffers to use for each outgoing/incoming channel (subpartition/input channel)." + + "In credit-based flow control mode, this indicates how many credits are exclusive in each input channel. It should be" + + " configured at least 2 for good performance. 1 buffer is for receiving in-flight data in the subpartition and 1 buffer is" + + " for parallel serialization."); /** * Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). @@ -277,7 +280,12 @@ public class TaskManagerOptions { public static final ConfigOption<Integer> NETWORK_EXTRA_BUFFERS_PER_GATE = key("taskmanager.network.memory.floating-buffers-per-gate") .defaultValue(8) - .withDescription("Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate)."); + .withDescription("Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate)." + + " In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels." + + " The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can" + + " help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be" + + " increased in case of higher round trip times between nodes and/or larger number of machines in the cluster."); + /** * Minimum backoff for partition requests of input channels. @@ -307,7 +315,7 @@ public class TaskManagerOptions { .withDescription("Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue lengths."); /** - * Config parameter defining whether to enable credit-based flow control or not. + * Boolean flag to enable/disable network credit-based flow control. * * @deprecated Will be removed for Flink 1.6 when the old code will be dropped in favour of * credit-based flow control. @@ -315,7 +323,8 @@ public class TaskManagerOptions { @Deprecated public static final ConfigOption<Boolean> NETWORK_CREDIT_BASED_FLOW_CONTROL_ENABLED = key("taskmanager.network.credit-based-flow-control.enabled") - .defaultValue(true); + .defaultValue(true) + .withDescription("Boolean flag to enable/disable network credit-based flow control."); /** * Config parameter defining whether to spill data for channels with barrier or not in exactly-once
