hudi-agent commented on code in PR #19641:
URL: https://github.com/apache/hudi/pull/19641#discussion_r3789558559
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java:
##########
@@ -349,6 +349,61 @@ public class FlinkOptions extends HoodieConfig {
.noDefaultValue()
.withDescription("Parallelism of tasks that do the index writing,
default is the parallelism of the execution environment");
+ // ------------------------------------------------------------------------
+ // RocksDB Partitioned RLI Cache Options
+ // ------------------------------------------------------------------------
+
+ @AdvancedConfig
+ public static final ConfigOption<Boolean> INDEX_RLI_ROCKSDB_CACHE_ENABLED =
ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Enable RocksDB-based partitioned cache for record
level index lookups, default false.");
+
+ @AdvancedConfig
+ public static final ConfigOption<String> INDEX_RLI_ROCKSDB_CACHE_BASE_PATH =
ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.base.path")
+ .stringType()
+ .defaultValue("/tmp/hudi-index-cache")
+ .withDescription("Local directory for RocksDB partitioned record level
index cache data.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Integer>
INDEX_RLI_ROCKSDB_CACHE_BOOTSTRAP_DAYS = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.bootstrap.days")
+ .intType()
+ .defaultValue(7)
+ .withDescription("Number of days of Partitioned Record Index to load
during bootstrap. Only partitions "
+ + "within this window are pre-loaded; older partitions are loaded on
demand when updates are observed.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Long>
INDEX_RLI_ROCKSDB_CACHE_PARTITION_TTL_HOURS = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.partition.ttl.hours")
+ .longType()
+ .defaultValue(168L) // default 7 days
+ .withDescription("TTL for partition column families in the RocksDB
partitioned record level index cache, default 168 hours (7 days).");
+
+ @AdvancedConfig
+ public static final ConfigOption<Long>
INDEX_RLI_ROCKSDB_CACHE_BLOCK_CACHE_MB = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.block.cache.mb")
+ .longType()
+ .defaultValue(256L)
+ .withDescription("RocksDB block cache size (off-heap) in MB for the
partitioned record level index cache, default 256.");
+
+ @AdvancedConfig
+ public static final ConfigOption<String>
INDEX_RLI_ROCKSDB_CACHE_COMPACTION_STYLE = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.compaction.style")
+ .stringType()
+ .defaultValue("LEVEL")
+ .withDescription("RocksDB compaction style for the partitioned record
level index cache, default LEVEL.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Boolean>
INDEX_RLI_ROCKSDB_CACHE_INVALIDATE_ON_REPLACE_COMMIT = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.invalidate.on.replacecommit")
Review Comment:
🤖 nit: `replacecommit` is missing a dot separator — could you rename this to
`hoodie.record.index.cache.rocksdb.invalidate.on.replace.commit` to stay
consistent with the dot-separated key convention used everywhere else?
<sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag
quality.</i></sub>
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java:
##########
@@ -349,6 +349,61 @@ public class FlinkOptions extends HoodieConfig {
.noDefaultValue()
.withDescription("Parallelism of tasks that do the index writing,
default is the parallelism of the execution environment");
+ // ------------------------------------------------------------------------
+ // RocksDB Partitioned RLI Cache Options
+ // ------------------------------------------------------------------------
+
+ @AdvancedConfig
+ public static final ConfigOption<Boolean> INDEX_RLI_ROCKSDB_CACHE_ENABLED =
ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Enable RocksDB-based partitioned cache for record
level index lookups, default false.");
+
+ @AdvancedConfig
+ public static final ConfigOption<String> INDEX_RLI_ROCKSDB_CACHE_BASE_PATH =
ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.base.path")
+ .stringType()
+ .defaultValue("/tmp/hudi-index-cache")
+ .withDescription("Local directory for RocksDB partitioned record level
index cache data.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Integer>
INDEX_RLI_ROCKSDB_CACHE_BOOTSTRAP_DAYS = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.bootstrap.days")
+ .intType()
+ .defaultValue(7)
+ .withDescription("Number of days of Partitioned Record Index to load
during bootstrap. Only partitions "
+ + "within this window are pre-loaded; older partitions are loaded on
demand when updates are observed.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Long>
INDEX_RLI_ROCKSDB_CACHE_PARTITION_TTL_HOURS = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.partition.ttl.hours")
+ .longType()
+ .defaultValue(168L) // default 7 days
Review Comment:
🤖 nit: the `// default 7 days` inline comment is already covered by the
description string — could you drop it to avoid duplication that can drift if
the default changes?
<sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag
quality.</i></sub>
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java:
##########
@@ -349,6 +349,61 @@ public class FlinkOptions extends HoodieConfig {
.noDefaultValue()
.withDescription("Parallelism of tasks that do the index writing,
default is the parallelism of the execution environment");
+ // ------------------------------------------------------------------------
+ // RocksDB Partitioned RLI Cache Options
+ // ------------------------------------------------------------------------
+
+ @AdvancedConfig
+ public static final ConfigOption<Boolean> INDEX_RLI_ROCKSDB_CACHE_ENABLED =
ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Enable RocksDB-based partitioned cache for record
level index lookups, default false.");
+
+ @AdvancedConfig
+ public static final ConfigOption<String> INDEX_RLI_ROCKSDB_CACHE_BASE_PATH =
ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.base.path")
+ .stringType()
+ .defaultValue("/tmp/hudi-index-cache")
+ .withDescription("Local directory for RocksDB partitioned record level
index cache data.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Integer>
INDEX_RLI_ROCKSDB_CACHE_BOOTSTRAP_DAYS = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.bootstrap.days")
+ .intType()
+ .defaultValue(7)
+ .withDescription("Number of days of Partitioned Record Index to load
during bootstrap. Only partitions "
+ + "within this window are pre-loaded; older partitions are loaded on
demand when updates are observed.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Long>
INDEX_RLI_ROCKSDB_CACHE_PARTITION_TTL_HOURS = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.partition.ttl.hours")
+ .longType()
+ .defaultValue(168L) // default 7 days
+ .withDescription("TTL for partition column families in the RocksDB
partitioned record level index cache, default 168 hours (7 days).");
+
+ @AdvancedConfig
+ public static final ConfigOption<Long>
INDEX_RLI_ROCKSDB_CACHE_BLOCK_CACHE_MB = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.block.cache.mb")
+ .longType()
+ .defaultValue(256L)
+ .withDescription("RocksDB block cache size (off-heap) in MB for the
partitioned record level index cache, default 256.");
+
+ @AdvancedConfig
+ public static final ConfigOption<String>
INDEX_RLI_ROCKSDB_CACHE_COMPACTION_STYLE = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.compaction.style")
+ .stringType()
+ .defaultValue("LEVEL")
+ .withDescription("RocksDB compaction style for the partitioned record
level index cache, default LEVEL.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Boolean>
INDEX_RLI_ROCKSDB_CACHE_INVALIDATE_ON_REPLACE_COMMIT = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.invalidate.on.replacecommit")
Review Comment:
🤖 Worth noting that `replacecommit` isn't an arbitrary run-together — it
mirrors Hudi's own action name, `HoodieTimeline.REPLACE_COMMIT_ACTION =
"replacecommit"` (hudi-common .../HoodieTimeline.java). Since this config keys
off that action, keeping it as one token stays consistent with how the action
is spelled everywhere else in the timeline; `replace.commit` would actually
diverge from it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
