This is an automated email from the ASF dual-hosted git repository. hangxiang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
commit 26e61931de6c24370162eb9777e3433b23004995 Author: Hangxiang Yu <master...@gmail.com> AuthorDate: Sat Jun 8 21:06:38 2024 +0800 [FLINK-34482][checkpoint] Refactor checkpoint section in doc --- docs/content.zh/docs/deployment/config.md | 10 ++++- docs/content/docs/deployment/config.md | 10 ++++- ...tion.html => common_checkpointing_section.html} | 24 ++++------- .../generated/common_state_backends_section.html | 48 ---------------------- ...tion.html => expert_checkpointing_section.html} | 0 .../flink/annotation/docs/Documentation.java | 3 +- .../flink/configuration/CheckpointingOptions.java | 21 +++++----- .../flink/configuration/StateRecoveryOptions.java | 1 - 8 files changed, 38 insertions(+), 79 deletions(-) diff --git a/docs/content.zh/docs/deployment/config.md b/docs/content.zh/docs/deployment/config.md index 896ea3102ca..8fbf550db8f 100644 --- a/docs/content.zh/docs/deployment/config.md +++ b/docs/content.zh/docs/deployment/config.md @@ -244,8 +244,14 @@ These options control the basic setup of state backends and checkpointing behavi The options are only relevant for jobs/applications executing in a continuous streaming fashion. Jobs/applications executing in a batch fashion do not use state backends and checkpoints, but different internal data structures that are optimized for batch processing. +**State Backends** + {{< generated/common_state_backends_section >}} +**Checkpoints** + +{{< generated/common_checkpointing_section >}} + ### High Availability High-availability here refers to the ability of the JobManager process to recover from failures. @@ -454,9 +460,9 @@ Please refer to the [Debugging Classloading Docs]({{< ref "docs/ops/debugging/de {{< generated/expert_debugging_and_tuning_section >}} -### Advanced State Backends Options +### Advanced Checkpointing Options -{{< generated/expert_state_backends_section >}} +{{< generated/expert_checkpointing_section >}} ### State Latency Tracking Options diff --git a/docs/content/docs/deployment/config.md b/docs/content/docs/deployment/config.md index ada52f3b63f..7c149528963 100644 --- a/docs/content/docs/deployment/config.md +++ b/docs/content/docs/deployment/config.md @@ -245,8 +245,14 @@ These options control the basic setup of state backends and checkpointing behavi The options are only relevant for jobs/applications executing in a continuous streaming fashion. Jobs/applications executing in a batch fashion do not use state backends and checkpoints, but different internal data structures that are optimized for batch processing. +**State Backends** + {{< generated/common_state_backends_section >}} +**Checkpoints** + +{{< generated/common_checkpointing_section >}} + ### High Availability High-availability here refers to the ability of the JobManager process to recover from failures. @@ -456,9 +462,9 @@ Please refer to the [Debugging Classloading Docs]({{< ref "docs/ops/debugging/de {{< generated/expert_debugging_and_tuning_section >}} -### Advanced State Backends Options +### Advanced Checkpointing Options -{{< generated/expert_state_backends_section >}} +{{< generated/expert_checkpointing_section >}} ### State Latency Tracking Options diff --git a/docs/layouts/shortcodes/generated/common_state_backends_section.html b/docs/layouts/shortcodes/generated/common_checkpointing_section.html similarity index 81% copy from docs/layouts/shortcodes/generated/common_state_backends_section.html copy to docs/layouts/shortcodes/generated/common_checkpointing_section.html index 7fe3a46207a..9db118174a3 100644 --- a/docs/layouts/shortcodes/generated/common_state_backends_section.html +++ b/docs/layouts/shortcodes/generated/common_checkpointing_section.html @@ -9,10 +9,10 @@ </thead> <tbody> <tr> - <td><h5>state.backend.type</h5></td> - <td style="word-wrap: break-word;">"hashmap"</td> + <td><h5>execution.checkpointing.storage</h5></td> + <td style="word-wrap: break-word;">(none)</td> <td>String</td> - <td>The state backend to be used to store state.<br />The implementation can be specified either via their shortcut name, or via the class name of a <code class="highlighter-rouge">StateBackendFactory</code>. If a factory is specified it is instantiated via its zero argument constructor and its <code class="highlighter-rouge">StateBackendFactory#createFromConfig(ReadableConfig, ClassLoader)</code> method is called.<br />Recognized shortcut names are 'hashmap' and 'rocksdb'.</td> + <td>The checkpoint storage implementation to be used to checkpoint state.<br />The implementation can be specified either via their shortcut name, or via the class name of a <code class="highlighter-rouge">CheckpointStorageFactory</code>. If a factory is specified it is instantiated via its zero argument constructor and its <code class="highlighter-rouge">CheckpointStorageFactory#createFromConfig(ReadableConfig, ClassLoader)</code> method is called.<br />Recognized shortcut [...] </tr> <tr> <td><h5>execution.checkpointing.dir</h5></td> @@ -20,12 +20,6 @@ <td>String</td> <td>The default directory used for storing the data files and meta data of checkpoints in a Flink supported filesystem. The storage path must be accessible from all participating processes/nodes(i.e. all TaskManagers and JobManagers). If the 'execution.checkpointing.storage' is set to 'jobmanager', only the meta data of checkpoints will be stored in this directory.</td> </tr> - <tr> - <td><h5>execution.checkpointing.storage</h5></td> - <td style="word-wrap: break-word;">(none)</td> - <td>String</td> - <td>The checkpoint storage implementation to be used to checkpoint state.<br />The implementation can be specified either via their shortcut name, or via the class name of a <code class="highlighter-rouge">CheckpointStorageFactory</code>. If a factory is specified it is instantiated via its zero argument constructor and its <code class="highlighter-rouge">CheckpointStorageFactory#createFromConfig(ReadableConfig, ClassLoader)</code> method is called.<br />Recognized shortcut [...] - </tr> <tr> <td><h5>execution.checkpointing.savepoint-dir</h5></td> <td style="word-wrap: break-word;">(none)</td> @@ -50,17 +44,17 @@ <td>String</td> <td>The config parameter defining the root directories for storing file-based state for local recovery. Local recovery currently only covers keyed state backends. If not configured it will default to <WORKING_DIR>/localState. The <WORKING_DIR> can be configured via <code class="highlighter-rouge">process.taskmanager.working-dir</code></td> </tr> + <tr> + <td><h5>execution.checkpointing.local-backup.enabled</h5></td> + <td style="word-wrap: break-word;">false</td> + <td>Boolean</td> + <td>This option configures local backup for the state backend, which indicates whether to make backup checkpoint on local disk. If not configured, fallback to execution.state-recovery.from-local. By default, local backup is deactivated. Local backup currently only covers keyed state backends (including both the EmbeddedRocksDBStateBackend and the HashMapStateBackend).</td> + </tr> <tr> <td><h5>execution.checkpointing.num-retained</h5></td> <td style="word-wrap: break-word;">1</td> <td>Integer</td> <td>The maximum number of completed checkpoints to retain.</td> </tr> - <tr> - <td><h5>execution.state-recovery.from-local</h5></td> - <td style="word-wrap: break-word;">false</td> - <td>Boolean</td> - <td>This option configures local recovery for the state backend, which indicates whether to recovery from local snapshot.By default, local recovery is deactivated. Local recovery currently only covers keyed state backends (including both the EmbeddedRocksDBStateBackend and the HashMapStateBackend)."</td> - </tr> </tbody> </table> diff --git a/docs/layouts/shortcodes/generated/common_state_backends_section.html b/docs/layouts/shortcodes/generated/common_state_backends_section.html index 7fe3a46207a..dd6106e7287 100644 --- a/docs/layouts/shortcodes/generated/common_state_backends_section.html +++ b/docs/layouts/shortcodes/generated/common_state_backends_section.html @@ -14,53 +14,5 @@ <td>String</td> <td>The state backend to be used to store state.<br />The implementation can be specified either via their shortcut name, or via the class name of a <code class="highlighter-rouge">StateBackendFactory</code>. If a factory is specified it is instantiated via its zero argument constructor and its <code class="highlighter-rouge">StateBackendFactory#createFromConfig(ReadableConfig, ClassLoader)</code> method is called.<br />Recognized shortcut names are 'hashmap' and 'rocksdb'.</td> </tr> - <tr> - <td><h5>execution.checkpointing.dir</h5></td> - <td style="word-wrap: break-word;">(none)</td> - <td>String</td> - <td>The default directory used for storing the data files and meta data of checkpoints in a Flink supported filesystem. The storage path must be accessible from all participating processes/nodes(i.e. all TaskManagers and JobManagers). If the 'execution.checkpointing.storage' is set to 'jobmanager', only the meta data of checkpoints will be stored in this directory.</td> - </tr> - <tr> - <td><h5>execution.checkpointing.storage</h5></td> - <td style="word-wrap: break-word;">(none)</td> - <td>String</td> - <td>The checkpoint storage implementation to be used to checkpoint state.<br />The implementation can be specified either via their shortcut name, or via the class name of a <code class="highlighter-rouge">CheckpointStorageFactory</code>. If a factory is specified it is instantiated via its zero argument constructor and its <code class="highlighter-rouge">CheckpointStorageFactory#createFromConfig(ReadableConfig, ClassLoader)</code> method is called.<br />Recognized shortcut [...] - </tr> - <tr> - <td><h5>execution.checkpointing.savepoint-dir</h5></td> - <td style="word-wrap: break-word;">(none)</td> - <td>String</td> - <td>The default directory for savepoints. Used by the state backends that write savepoints to file systems (HashMapStateBackend, EmbeddedRocksDBStateBackend).</td> - </tr> - <tr> - <td><h5>execution.checkpointing.cleaner.parallel-mode</h5></td> - <td style="word-wrap: break-word;">true</td> - <td>Boolean</td> - <td>Option whether to discard a checkpoint's states in parallel using the ExecutorService passed into the cleaner</td> - </tr> - <tr> - <td><h5>execution.checkpointing.incremental</h5></td> - <td style="word-wrap: break-word;">false</td> - <td>Boolean</td> - <td>Option whether to create incremental checkpoints, if possible. For an incremental checkpoint, only a diff from the previous checkpoint is stored, rather than the complete checkpoint state. Once enabled, the state size shown in web UI or fetched from rest API only represents the delta checkpoint size instead of full checkpoint size. Some state backends may not support incremental checkpoints and ignore this option.</td> - </tr> - <tr> - <td><h5>execution.checkpointing.local-backup.dirs</h5></td> - <td style="word-wrap: break-word;">(none)</td> - <td>String</td> - <td>The config parameter defining the root directories for storing file-based state for local recovery. Local recovery currently only covers keyed state backends. If not configured it will default to <WORKING_DIR>/localState. The <WORKING_DIR> can be configured via <code class="highlighter-rouge">process.taskmanager.working-dir</code></td> - </tr> - <tr> - <td><h5>execution.checkpointing.num-retained</h5></td> - <td style="word-wrap: break-word;">1</td> - <td>Integer</td> - <td>The maximum number of completed checkpoints to retain.</td> - </tr> - <tr> - <td><h5>execution.state-recovery.from-local</h5></td> - <td style="word-wrap: break-word;">false</td> - <td>Boolean</td> - <td>This option configures local recovery for the state backend, which indicates whether to recovery from local snapshot.By default, local recovery is deactivated. Local recovery currently only covers keyed state backends (including both the EmbeddedRocksDBStateBackend and the HashMapStateBackend)."</td> - </tr> </tbody> </table> diff --git a/docs/layouts/shortcodes/generated/expert_state_backends_section.html b/docs/layouts/shortcodes/generated/expert_checkpointing_section.html similarity index 100% rename from docs/layouts/shortcodes/generated/expert_state_backends_section.html rename to docs/layouts/shortcodes/generated/expert_checkpointing_section.html diff --git a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java index 348f520710c..8b766624152 100644 --- a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java +++ b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java @@ -62,6 +62,7 @@ public final class Documentation { public static final String COMMON_HOST_PORT = "common_host_port"; public static final String COMMON_STATE_BACKENDS = "common_state_backends"; + public static final String COMMON_CHECKPOINTING = "common_checkpointing"; public static final String COMMON_HIGH_AVAILABILITY = "common_high_availability"; public static final String COMMON_HIGH_AVAILABILITY_ZOOKEEPER = "common_high_availability_zk"; @@ -85,7 +86,7 @@ public final class Documentation { public static final String EXPERT_DEBUGGING_AND_TUNING = "expert_debugging_and_tuning"; public static final String EXPERT_SCHEDULING = "expert_scheduling"; public static final String EXPERT_FAULT_TOLERANCE = "expert_fault_tolerance"; - public static final String EXPERT_STATE_BACKENDS = "expert_state_backends"; + public static final String EXPERT_CHECKPOINTING = "expert_checkpointing"; public static final String EXPERT_REST = "expert_rest"; public static final String EXPERT_HIGH_AVAILABILITY = "expert_high_availability"; public static final String EXPERT_ZOOKEEPER_HIGH_AVAILABILITY = diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java index 1311c554f4a..4bb30c1c563 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java @@ -97,7 +97,7 @@ public class CheckpointingOptions { * to {@link #CHECKPOINTS_DIRECTORY}, and the checkpoint meta data and actual program state * will both be persisted to the path. */ - @Documentation.Section(value = Documentation.Sections.COMMON_STATE_BACKENDS, position = 2) + @Documentation.Section(value = Documentation.Sections.COMMON_CHECKPOINTING, position = 2) public static final ConfigOption<String> CHECKPOINT_STORAGE = ConfigOptions.key("execution.checkpointing.storage") .stringType() @@ -132,7 +132,7 @@ public class CheckpointingOptions { .build()); /** The maximum number of completed checkpoints to retain. */ - @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS) + @Documentation.Section(Documentation.Sections.COMMON_CHECKPOINTING) public static final ConfigOption<Integer> MAX_RETAINED_CHECKPOINTS = ConfigOptions.key("execution.checkpointing.num-retained") .intType() @@ -144,7 +144,7 @@ public class CheckpointingOptions { * operator states are discarded in parallel using the ExecutorService passed to the cleaner. * This speeds up checkpoints cleaning, but adds load to the IO. */ - @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS) + @Documentation.Section(Documentation.Sections.COMMON_CHECKPOINTING) public static final ConfigOption<Boolean> CLEANER_PARALLEL_MODE = ConfigOptions.key("execution.checkpointing.cleaner.parallel-mode") .booleanType() @@ -172,7 +172,7 @@ public class CheckpointingOptions { * * <p>Some state backends may not support incremental checkpoints and ignore this option. */ - @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS) + @Documentation.Section(Documentation.Sections.COMMON_CHECKPOINTING) public static final ConfigOption<Boolean> INCREMENTAL_CHECKPOINTS = ConfigOptions.key("execution.checkpointing.incremental") .booleanType() @@ -214,7 +214,7 @@ public class CheckpointingOptions { * <p>Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend * does not support local recovery and ignore this option. */ - @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS) + @Documentation.Section(Documentation.Sections.COMMON_CHECKPOINTING) public static final ConfigOption<String> LOCAL_RECOVERY_TASK_MANAGER_STATE_ROOT_DIRS = ConfigOptions.key("execution.checkpointing.local-backup.dirs") .stringType() @@ -241,7 +241,7 @@ public class CheckpointingOptions { * The default directory for savepoints. Used by the state backends that write savepoints to * file systems (HashMapStateBackend, EmbeddedRocksDBStateBackend). */ - @Documentation.Section(value = Documentation.Sections.COMMON_STATE_BACKENDS, position = 3) + @Documentation.Section(value = Documentation.Sections.COMMON_CHECKPOINTING, position = 4) public static final ConfigOption<String> SAVEPOINT_DIRECTORY = ConfigOptions.key("execution.checkpointing.savepoint-dir") .stringType() @@ -257,7 +257,7 @@ public class CheckpointingOptions { * processes/nodes(i.e. all TaskManagers and JobManagers). If {@link #CHECKPOINT_STORAGE} is set * to 'jobmanager', only the meta data of checkpoints will be stored in this directory. */ - @Documentation.Section(value = Documentation.Sections.COMMON_STATE_BACKENDS, position = 2) + @Documentation.Section(value = Documentation.Sections.COMMON_CHECKPOINTING, position = 3) public static final ConfigOption<String> CHECKPOINTS_DIRECTORY = ConfigOptions.key("execution.checkpointing.dir") .stringType() @@ -276,7 +276,7 @@ public class CheckpointingOptions { * checkpoint directory at the same time. If this value is set to false, pay attention not to * run several jobs with the same directory simultaneously. */ - @Documentation.Section(Documentation.Sections.EXPERT_STATE_BACKENDS) + @Documentation.Section(Documentation.Sections.EXPERT_CHECKPOINTING) public static final ConfigOption<Boolean> CREATE_CHECKPOINT_SUB_DIR = ConfigOptions.key("execution.checkpointing.create-subdir") .booleanType() @@ -301,7 +301,7 @@ public class CheckpointingOptions { * The minimum size of state data files. All state chunks smaller than that are stored inline in * the root checkpoint metadata file. */ - @Documentation.Section(Documentation.Sections.EXPERT_STATE_BACKENDS) + @Documentation.Section(Documentation.Sections.EXPERT_CHECKPOINTING) public static final ConfigOption<MemorySize> FS_SMALL_FILE_THRESHOLD = ConfigOptions.key("execution.checkpointing.data-inline-threshold") .memoryType() @@ -316,7 +316,7 @@ public class CheckpointingOptions { /** * The default size of the write buffer for the checkpoint streams that write to file systems. */ - @Documentation.Section(Documentation.Sections.EXPERT_STATE_BACKENDS) + @Documentation.Section(Documentation.Sections.EXPERT_CHECKPOINTING) public static final ConfigOption<Integer> FS_WRITE_BUFFER_SIZE = ConfigOptions.key("execution.checkpointing.write-buffer-size") .intType() @@ -337,6 +337,7 @@ public class CheckpointingOptions { * currently only covers keyed state backends (including both the EmbeddedRocksDBStateBackend * and the HashMapStateBackend). */ + @Documentation.Section(value = Documentation.Sections.COMMON_CHECKPOINTING) public static final ConfigOption<Boolean> LOCAL_BACKUP_ENABLED = ConfigOptions.key("execution.checkpointing.local-backup.enabled") .booleanType() diff --git a/flink-core/src/main/java/org/apache/flink/configuration/StateRecoveryOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/StateRecoveryOptions.java index c19329c5d92..3dd0d339602 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/StateRecoveryOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/StateRecoveryOptions.java @@ -105,7 +105,6 @@ public class StateRecoveryOptions { * <p>Local recovery currently only covers keyed state backends (including both the * EmbeddedRocksDBStateBackend and the HashMapStateBackend). */ - @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS) public static final ConfigOption<Boolean> LOCAL_RECOVERY = ConfigOptions.key("execution.state-recovery.from-local") .booleanType()