This is an automated email from the ASF dual-hosted git repository.
hangxiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 26e61931de6c24370162eb9777e3433b23004995
Author: Hangxiang Yu <master...@gmail.com>
AuthorDate: Sat Jun 8 21:06:38 2024 +0800
[FLINK-34482][checkpoint] Refactor checkpoint section in doc
---
docs/content.zh/docs/deployment/config.md | 10 ++++-
docs/content/docs/deployment/config.md | 10 ++++-
...tion.html => common_checkpointing_section.html} | 24 ++++-------
.../generated/common_state_backends_section.html | 48 ----------------------
...tion.html => expert_checkpointing_section.html} | 0
.../flink/annotation/docs/Documentation.java | 3 +-
.../flink/configuration/CheckpointingOptions.java | 21 +++++-----
.../flink/configuration/StateRecoveryOptions.java | 1 -
8 files changed, 38 insertions(+), 79 deletions(-)
diff --git a/docs/content.zh/docs/deployment/config.md
b/docs/content.zh/docs/deployment/config.md
index 896ea3102ca..8fbf550db8f 100644
--- a/docs/content.zh/docs/deployment/config.md
+++ b/docs/content.zh/docs/deployment/config.md
@@ -244,8 +244,14 @@ These options control the basic setup of state backends
and checkpointing behavi
The options are only relevant for jobs/applications executing in a continuous
streaming fashion.
Jobs/applications executing in a batch fashion do not use state backends and
checkpoints, but different internal data structures that are optimized for
batch processing.
+**State Backends**
+
{{< generated/common_state_backends_section >}}
+**Checkpoints**
+
+{{< generated/common_checkpointing_section >}}
+
### High Availability
High-availability here refers to the ability of the JobManager process to
recover from failures.
@@ -454,9 +460,9 @@ Please refer to the [Debugging Classloading Docs]({{< ref
"docs/ops/debugging/de
{{< generated/expert_debugging_and_tuning_section >}}
-### Advanced State Backends Options
+### Advanced Checkpointing Options
-{{< generated/expert_state_backends_section >}}
+{{< generated/expert_checkpointing_section >}}
### State Latency Tracking Options
diff --git a/docs/content/docs/deployment/config.md
b/docs/content/docs/deployment/config.md
index ada52f3b63f..7c149528963 100644
--- a/docs/content/docs/deployment/config.md
+++ b/docs/content/docs/deployment/config.md
@@ -245,8 +245,14 @@ These options control the basic setup of state backends
and checkpointing behavi
The options are only relevant for jobs/applications executing in a continuous
streaming fashion.
Jobs/applications executing in a batch fashion do not use state backends and
checkpoints, but different internal data structures that are optimized for
batch processing.
+**State Backends**
+
{{< generated/common_state_backends_section >}}
+**Checkpoints**
+
+{{< generated/common_checkpointing_section >}}
+
### High Availability
High-availability here refers to the ability of the JobManager process to
recover from failures.
@@ -456,9 +462,9 @@ Please refer to the [Debugging Classloading Docs]({{< ref
"docs/ops/debugging/de
{{< generated/expert_debugging_and_tuning_section >}}
-### Advanced State Backends Options
+### Advanced Checkpointing Options
-{{< generated/expert_state_backends_section >}}
+{{< generated/expert_checkpointing_section >}}
### State Latency Tracking Options
diff --git
a/docs/layouts/shortcodes/generated/common_state_backends_section.html
b/docs/layouts/shortcodes/generated/common_checkpointing_section.html
similarity index 81%
copy from docs/layouts/shortcodes/generated/common_state_backends_section.html
copy to docs/layouts/shortcodes/generated/common_checkpointing_section.html
index 7fe3a46207a..9db118174a3 100644
--- a/docs/layouts/shortcodes/generated/common_state_backends_section.html
+++ b/docs/layouts/shortcodes/generated/common_checkpointing_section.html
@@ -9,10 +9,10 @@
</thead>
<tbody>
<tr>
- <td><h5>state.backend.type</h5></td>
- <td style="word-wrap: break-word;">"hashmap"</td>
+ <td><h5>execution.checkpointing.storage</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
<td>String</td>
- <td>The state backend to be used to store state.<br />The
implementation can be specified either via their shortcut name, or via the
class name of a <code class="highlighter-rouge">StateBackendFactory</code>. If
a factory is specified it is instantiated via its zero argument constructor and
its <code
class="highlighter-rouge">StateBackendFactory#createFromConfig(ReadableConfig,
ClassLoader)</code> method is called.<br />Recognized shortcut names are
'hashmap' and 'rocksdb'.</td>
+ <td>The checkpoint storage implementation to be used to checkpoint
state.<br />The implementation can be specified either via their shortcut
name, or via the class name of a <code
class="highlighter-rouge">CheckpointStorageFactory</code>. If a factory is
specified it is instantiated via its zero argument constructor and its <code
class="highlighter-rouge">CheckpointStorageFactory#createFromConfig(ReadableConfig,
ClassLoader)</code> method is called.<br />Recognized shortcut [...]
</tr>
<tr>
<td><h5>execution.checkpointing.dir</h5></td>
@@ -20,12 +20,6 @@
<td>String</td>
<td>The default directory used for storing the data files and meta
data of checkpoints in a Flink supported filesystem. The storage path must be
accessible from all participating processes/nodes(i.e. all TaskManagers and
JobManagers). If the 'execution.checkpointing.storage' is set to 'jobmanager',
only the meta data of checkpoints will be stored in this directory.</td>
</tr>
- <tr>
- <td><h5>execution.checkpointing.storage</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The checkpoint storage implementation to be used to checkpoint
state.<br />The implementation can be specified either via their shortcut
name, or via the class name of a <code
class="highlighter-rouge">CheckpointStorageFactory</code>. If a factory is
specified it is instantiated via its zero argument constructor and its <code
class="highlighter-rouge">CheckpointStorageFactory#createFromConfig(ReadableConfig,
ClassLoader)</code> method is called.<br />Recognized shortcut [...]
- </tr>
<tr>
<td><h5>execution.checkpointing.savepoint-dir</h5></td>
<td style="word-wrap: break-word;">(none)</td>
@@ -50,17 +44,17 @@
<td>String</td>
<td>The config parameter defining the root directories for storing
file-based state for local recovery. Local recovery currently only covers keyed
state backends. If not configured it will default to
<WORKING_DIR>/localState. The <WORKING_DIR> can be configured via
<code class="highlighter-rouge">process.taskmanager.working-dir</code></td>
</tr>
+ <tr>
+ <td><h5>execution.checkpointing.local-backup.enabled</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>This option configures local backup for the state backend,
which indicates whether to make backup checkpoint on local disk. If not
configured, fallback to execution.state-recovery.from-local. By default, local
backup is deactivated. Local backup currently only covers keyed state backends
(including both the EmbeddedRocksDBStateBackend and the
HashMapStateBackend).</td>
+ </tr>
<tr>
<td><h5>execution.checkpointing.num-retained</h5></td>
<td style="word-wrap: break-word;">1</td>
<td>Integer</td>
<td>The maximum number of completed checkpoints to retain.</td>
</tr>
- <tr>
- <td><h5>execution.state-recovery.from-local</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>This option configures local recovery for the state backend,
which indicates whether to recovery from local snapshot.By default, local
recovery is deactivated. Local recovery currently only covers keyed state
backends (including both the EmbeddedRocksDBStateBackend and the
HashMapStateBackend)."</td>
- </tr>
</tbody>
</table>
diff --git
a/docs/layouts/shortcodes/generated/common_state_backends_section.html
b/docs/layouts/shortcodes/generated/common_state_backends_section.html
index 7fe3a46207a..dd6106e7287 100644
--- a/docs/layouts/shortcodes/generated/common_state_backends_section.html
+++ b/docs/layouts/shortcodes/generated/common_state_backends_section.html
@@ -14,53 +14,5 @@
<td>String</td>
<td>The state backend to be used to store state.<br />The
implementation can be specified either via their shortcut name, or via the
class name of a <code class="highlighter-rouge">StateBackendFactory</code>. If
a factory is specified it is instantiated via its zero argument constructor and
its <code
class="highlighter-rouge">StateBackendFactory#createFromConfig(ReadableConfig,
ClassLoader)</code> method is called.<br />Recognized shortcut names are
'hashmap' and 'rocksdb'.</td>
</tr>
- <tr>
- <td><h5>execution.checkpointing.dir</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The default directory used for storing the data files and meta
data of checkpoints in a Flink supported filesystem. The storage path must be
accessible from all participating processes/nodes(i.e. all TaskManagers and
JobManagers). If the 'execution.checkpointing.storage' is set to 'jobmanager',
only the meta data of checkpoints will be stored in this directory.</td>
- </tr>
- <tr>
- <td><h5>execution.checkpointing.storage</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The checkpoint storage implementation to be used to checkpoint
state.<br />The implementation can be specified either via their shortcut
name, or via the class name of a <code
class="highlighter-rouge">CheckpointStorageFactory</code>. If a factory is
specified it is instantiated via its zero argument constructor and its <code
class="highlighter-rouge">CheckpointStorageFactory#createFromConfig(ReadableConfig,
ClassLoader)</code> method is called.<br />Recognized shortcut [...]
- </tr>
- <tr>
- <td><h5>execution.checkpointing.savepoint-dir</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The default directory for savepoints. Used by the state
backends that write savepoints to file systems (HashMapStateBackend,
EmbeddedRocksDBStateBackend).</td>
- </tr>
- <tr>
- <td><h5>execution.checkpointing.cleaner.parallel-mode</h5></td>
- <td style="word-wrap: break-word;">true</td>
- <td>Boolean</td>
- <td>Option whether to discard a checkpoint's states in parallel
using the ExecutorService passed into the cleaner</td>
- </tr>
- <tr>
- <td><h5>execution.checkpointing.incremental</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>Option whether to create incremental checkpoints, if possible.
For an incremental checkpoint, only a diff from the previous checkpoint is
stored, rather than the complete checkpoint state. Once enabled, the state size
shown in web UI or fetched from rest API only represents the delta checkpoint
size instead of full checkpoint size. Some state backends may not support
incremental checkpoints and ignore this option.</td>
- </tr>
- <tr>
- <td><h5>execution.checkpointing.local-backup.dirs</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The config parameter defining the root directories for storing
file-based state for local recovery. Local recovery currently only covers keyed
state backends. If not configured it will default to
<WORKING_DIR>/localState. The <WORKING_DIR> can be configured via
<code class="highlighter-rouge">process.taskmanager.working-dir</code></td>
- </tr>
- <tr>
- <td><h5>execution.checkpointing.num-retained</h5></td>
- <td style="word-wrap: break-word;">1</td>
- <td>Integer</td>
- <td>The maximum number of completed checkpoints to retain.</td>
- </tr>
- <tr>
- <td><h5>execution.state-recovery.from-local</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>This option configures local recovery for the state backend,
which indicates whether to recovery from local snapshot.By default, local
recovery is deactivated. Local recovery currently only covers keyed state
backends (including both the EmbeddedRocksDBStateBackend and the
HashMapStateBackend)."</td>
- </tr>
</tbody>
</table>
diff --git
a/docs/layouts/shortcodes/generated/expert_state_backends_section.html
b/docs/layouts/shortcodes/generated/expert_checkpointing_section.html
similarity index 100%
rename from docs/layouts/shortcodes/generated/expert_state_backends_section.html
rename to docs/layouts/shortcodes/generated/expert_checkpointing_section.html
diff --git
a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
index 348f520710c..8b766624152 100644
---
a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
+++
b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
@@ -62,6 +62,7 @@ public final class Documentation {
public static final String COMMON_HOST_PORT = "common_host_port";
public static final String COMMON_STATE_BACKENDS =
"common_state_backends";
+ public static final String COMMON_CHECKPOINTING =
"common_checkpointing";
public static final String COMMON_HIGH_AVAILABILITY =
"common_high_availability";
public static final String COMMON_HIGH_AVAILABILITY_ZOOKEEPER =
"common_high_availability_zk";
@@ -85,7 +86,7 @@ public final class Documentation {
public static final String EXPERT_DEBUGGING_AND_TUNING =
"expert_debugging_and_tuning";
public static final String EXPERT_SCHEDULING = "expert_scheduling";
public static final String EXPERT_FAULT_TOLERANCE =
"expert_fault_tolerance";
- public static final String EXPERT_STATE_BACKENDS =
"expert_state_backends";
+ public static final String EXPERT_CHECKPOINTING =
"expert_checkpointing";
public static final String EXPERT_REST = "expert_rest";
public static final String EXPERT_HIGH_AVAILABILITY =
"expert_high_availability";
public static final String EXPERT_ZOOKEEPER_HIGH_AVAILABILITY =
diff --git
a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java
b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java
index 1311c554f4a..4bb30c1c563 100644
---
a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java
+++
b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java
@@ -97,7 +97,7 @@ public class CheckpointingOptions {
* to {@link #CHECKPOINTS_DIRECTORY}, and the checkpoint meta data and
actual program state
* will both be persisted to the path.
*/
- @Documentation.Section(value =
Documentation.Sections.COMMON_STATE_BACKENDS, position = 2)
+ @Documentation.Section(value =
Documentation.Sections.COMMON_CHECKPOINTING, position = 2)
public static final ConfigOption<String> CHECKPOINT_STORAGE =
ConfigOptions.key("execution.checkpointing.storage")
.stringType()
@@ -132,7 +132,7 @@ public class CheckpointingOptions {
.build());
/** The maximum number of completed checkpoints to retain. */
- @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS)
+ @Documentation.Section(Documentation.Sections.COMMON_CHECKPOINTING)
public static final ConfigOption<Integer> MAX_RETAINED_CHECKPOINTS =
ConfigOptions.key("execution.checkpointing.num-retained")
.intType()
@@ -144,7 +144,7 @@ public class CheckpointingOptions {
* operator states are discarded in parallel using the ExecutorService
passed to the cleaner.
* This speeds up checkpoints cleaning, but adds load to the IO.
*/
- @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS)
+ @Documentation.Section(Documentation.Sections.COMMON_CHECKPOINTING)
public static final ConfigOption<Boolean> CLEANER_PARALLEL_MODE =
ConfigOptions.key("execution.checkpointing.cleaner.parallel-mode")
.booleanType()
@@ -172,7 +172,7 @@ public class CheckpointingOptions {
*
* <p>Some state backends may not support incremental checkpoints and
ignore this option.
*/
- @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS)
+ @Documentation.Section(Documentation.Sections.COMMON_CHECKPOINTING)
public static final ConfigOption<Boolean> INCREMENTAL_CHECKPOINTS =
ConfigOptions.key("execution.checkpointing.incremental")
.booleanType()
@@ -214,7 +214,7 @@ public class CheckpointingOptions {
* <p>Local recovery currently only covers keyed state backends.
Currently, MemoryStateBackend
* does not support local recovery and ignore this option.
*/
- @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS)
+ @Documentation.Section(Documentation.Sections.COMMON_CHECKPOINTING)
public static final ConfigOption<String>
LOCAL_RECOVERY_TASK_MANAGER_STATE_ROOT_DIRS =
ConfigOptions.key("execution.checkpointing.local-backup.dirs")
.stringType()
@@ -241,7 +241,7 @@ public class CheckpointingOptions {
* The default directory for savepoints. Used by the state backends that
write savepoints to
* file systems (HashMapStateBackend, EmbeddedRocksDBStateBackend).
*/
- @Documentation.Section(value =
Documentation.Sections.COMMON_STATE_BACKENDS, position = 3)
+ @Documentation.Section(value =
Documentation.Sections.COMMON_CHECKPOINTING, position = 4)
public static final ConfigOption<String> SAVEPOINT_DIRECTORY =
ConfigOptions.key("execution.checkpointing.savepoint-dir")
.stringType()
@@ -257,7 +257,7 @@ public class CheckpointingOptions {
* processes/nodes(i.e. all TaskManagers and JobManagers). If {@link
#CHECKPOINT_STORAGE} is set
* to 'jobmanager', only the meta data of checkpoints will be stored in
this directory.
*/
- @Documentation.Section(value =
Documentation.Sections.COMMON_STATE_BACKENDS, position = 2)
+ @Documentation.Section(value =
Documentation.Sections.COMMON_CHECKPOINTING, position = 3)
public static final ConfigOption<String> CHECKPOINTS_DIRECTORY =
ConfigOptions.key("execution.checkpointing.dir")
.stringType()
@@ -276,7 +276,7 @@ public class CheckpointingOptions {
* checkpoint directory at the same time. If this value is set to false,
pay attention not to
* run several jobs with the same directory simultaneously.
*/
- @Documentation.Section(Documentation.Sections.EXPERT_STATE_BACKENDS)
+ @Documentation.Section(Documentation.Sections.EXPERT_CHECKPOINTING)
public static final ConfigOption<Boolean> CREATE_CHECKPOINT_SUB_DIR =
ConfigOptions.key("execution.checkpointing.create-subdir")
.booleanType()
@@ -301,7 +301,7 @@ public class CheckpointingOptions {
* The minimum size of state data files. All state chunks smaller than
that are stored inline in
* the root checkpoint metadata file.
*/
- @Documentation.Section(Documentation.Sections.EXPERT_STATE_BACKENDS)
+ @Documentation.Section(Documentation.Sections.EXPERT_CHECKPOINTING)
public static final ConfigOption<MemorySize> FS_SMALL_FILE_THRESHOLD =
ConfigOptions.key("execution.checkpointing.data-inline-threshold")
.memoryType()
@@ -316,7 +316,7 @@ public class CheckpointingOptions {
/**
* The default size of the write buffer for the checkpoint streams that
write to file systems.
*/
- @Documentation.Section(Documentation.Sections.EXPERT_STATE_BACKENDS)
+ @Documentation.Section(Documentation.Sections.EXPERT_CHECKPOINTING)
public static final ConfigOption<Integer> FS_WRITE_BUFFER_SIZE =
ConfigOptions.key("execution.checkpointing.write-buffer-size")
.intType()
@@ -337,6 +337,7 @@ public class CheckpointingOptions {
* currently only covers keyed state backends (including both the
EmbeddedRocksDBStateBackend
* and the HashMapStateBackend).
*/
+ @Documentation.Section(value = Documentation.Sections.COMMON_CHECKPOINTING)
public static final ConfigOption<Boolean> LOCAL_BACKUP_ENABLED =
ConfigOptions.key("execution.checkpointing.local-backup.enabled")
.booleanType()
diff --git
a/flink-core/src/main/java/org/apache/flink/configuration/StateRecoveryOptions.java
b/flink-core/src/main/java/org/apache/flink/configuration/StateRecoveryOptions.java
index c19329c5d92..3dd0d339602 100644
---
a/flink-core/src/main/java/org/apache/flink/configuration/StateRecoveryOptions.java
+++
b/flink-core/src/main/java/org/apache/flink/configuration/StateRecoveryOptions.java
@@ -105,7 +105,6 @@ public class StateRecoveryOptions {
* <p>Local recovery currently only covers keyed state backends (including
both the
* EmbeddedRocksDBStateBackend and the HashMapStateBackend).
*/
- @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS)
public static final ConfigOption<Boolean> LOCAL_RECOVERY =
ConfigOptions.key("execution.state-recovery.from-local")
.booleanType()
