This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 2cb9acc3da [core] Add option to always select level 0 files when
compacting (#5428)
2cb9acc3da is described below
commit 2cb9acc3da3f1e91d0c177d15203345412f42182
Author: tsreaper <[email protected]>
AuthorDate: Wed Apr 9 22:46:49 2025 +0800
[core] Add option to always select level 0 files when compacting (#5428)
---
.../shortcodes/generated/core_configuration.html | 6 +++++
.../main/java/org/apache/paimon/CoreOptions.java | 11 ++++++++
.../paimon/operation/KeyValueFileStoreWrite.java | 16 ++++++++----
.../operation/KeyValueFileStoreWriteTest.java | 29 +++++++++++-----------
4 files changed, 42 insertions(+), 20 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 840470be3a..2a63755528 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -152,6 +152,12 @@ under the License.
<td>String</td>
<td>Specifies the commit user prefix.</td>
</tr>
+ <tr>
+ <td><h5>compaction.force-up-level-0</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>If set to true, compaction strategy will always include all
level 0 files in candidates.</td>
+ </tr>
<tr>
<td><h5>compaction.max-size-amplification-percent</h5></td>
<td style="word-wrap: break-word;">200</td>
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index 58a58a199f..fc5f327406 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -583,6 +583,13 @@ public class CoreOptions implements Serializable {
"The size amplification is defined as the amount
(in percentage) of additional storage "
+ "needed to store a single byte of data
in the merge tree for changelog mode table.");
+ public static final ConfigOption<Boolean> COMPACTION_FORCE_UP_LEVEL_0 =
+ key("compaction.force-up-level-0")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "If set to true, compaction strategy will always
include all level 0 files in candidates.");
+
public static final ConfigOption<Integer> COMPACTION_SIZE_RATIO =
key("compaction.size-ratio")
.intType()
@@ -2157,6 +2164,10 @@ public class CoreOptions implements Serializable {
return options.get(COMPACTION_MAX_SIZE_AMPLIFICATION_PERCENT);
}
+ public boolean compactionForceUpLevel0() {
+ return options.get(COMPACTION_FORCE_UP_LEVEL_0);
+ }
+
public int sortedRunSizeRatio() {
return options.get(COMPACTION_SIZE_RATIO);
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
index 6ac09b86ef..c2f2f729df 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
@@ -241,11 +241,17 @@ public class KeyValueFileStoreWrite extends
MemoryFileStoreWrite<KeyValue> {
}
}
- return new UniversalCompaction(
- options.maxSizeAmplificationPercent(),
- options.sortedRunSizeRatio(),
- options.numSortedRunCompactionTrigger(),
- options.optimizedCompactionInterval());
+ UniversalCompaction universal =
+ new UniversalCompaction(
+ options.maxSizeAmplificationPercent(),
+ options.sortedRunSizeRatio(),
+ options.numSortedRunCompactionTrigger(),
+ options.optimizedCompactionInterval());
+ if (options.compactionForceUpLevel0()) {
+ return new ForceUpLevel0Compaction(universal);
+ } else {
+ return universal;
+ }
}
private CompactManager createCompactManager(
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreWriteTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreWriteTest.java
index 0098b44b91..606a290bcb 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreWriteTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreWriteTest.java
@@ -65,20 +65,7 @@ public class KeyValueFileStoreWriteTest {
Map<String, String> options = new HashMap<>();
options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true");
options.put(CoreOptions.LOOKUP_COMPACT.key(), "radical");
-
- KeyValueFileStoreWrite write = createWriteWithOptions(options);
- write.withIOManager(ioManager);
- TestKeyValueGenerator gen = new TestKeyValueGenerator();
-
- KeyValue keyValue = gen.next();
- AbstractFileStoreWrite.WriterContainer<KeyValue> writerContainer =
- write.createWriterContainer(gen.getPartition(keyValue), 1,
false);
- MergeTreeWriter writer = (MergeTreeWriter) writerContainer.writer;
- try (MergeTreeCompactManager compactManager =
- (MergeTreeCompactManager) writer.compactManager()) {
- CompactStrategy compactStrategy = compactManager.getStrategy();
-
assertThat(compactStrategy).isInstanceOf(ForceUpLevel0Compaction.class);
- }
+ assertCompactStrategy(options, ForceUpLevel0Compaction.class);
}
@Test
@@ -86,7 +73,19 @@ public class KeyValueFileStoreWriteTest {
Map<String, String> options = new HashMap<>();
options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true");
options.put(CoreOptions.LOOKUP_COMPACT.key(), "gentle");
+ assertCompactStrategy(options, UniversalCompaction.class);
+ }
+ @Test
+ public void testForceUpLevel0CompactStrategy() throws Exception {
+ Map<String, String> options = new HashMap<>();
+ options.put(CoreOptions.COMPACTION_FORCE_UP_LEVEL_0.key(), "true");
+ assertCompactStrategy(options, ForceUpLevel0Compaction.class);
+ }
+
+ private void assertCompactStrategy(
+ Map<String, String> options, Class<? extends CompactStrategy>
expected)
+ throws Exception {
KeyValueFileStoreWrite write = createWriteWithOptions(options);
write.withIOManager(ioManager);
TestKeyValueGenerator gen = new TestKeyValueGenerator();
@@ -98,7 +97,7 @@ public class KeyValueFileStoreWriteTest {
try (MergeTreeCompactManager compactManager =
(MergeTreeCompactManager) writer.compactManager()) {
CompactStrategy compactStrategy = compactManager.getStrategy();
-
assertThat(compactStrategy).isInstanceOf(UniversalCompaction.class);
+ assertThat(compactStrategy).isInstanceOf(expected);
}
}
