This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 15e6ad460 [core] modify the default value of "target-file-size" (#3721)
15e6ad460 is described below
commit 15e6ad46045dc4b933f012645c83482bb3e27d69
Author: wangwj <[email protected]>
AuthorDate: Thu Jul 11 14:52:25 2024 +0800
[core] modify the default value of "target-file-size" (#3721)
---
.../shortcodes/generated/core_configuration.html | 2 +-
.../main/java/org/apache/paimon/CoreOptions.java | 46 +++++++++++++++++++---
.../AppendOnlyTableCompactionCoordinator.java | 5 ++-
.../paimon/operation/AppendOnlyFileStoreWrite.java | 2 +-
.../paimon/operation/KeyValueFileStoreWrite.java | 4 +-
.../AppendOnlyTableCompactionCoordinatorTest.java | 15 ++++++-
.../mergetree/ChangelogMergeTreeRewriterTest.java | 3 +-
.../paimon/mergetree/ContainsLevelsTest.java | 3 +-
.../apache/paimon/mergetree/LookupLevelsTest.java | 3 +-
.../apache/paimon/mergetree/MergeTreeTestBase.java | 6 +--
10 files changed, 67 insertions(+), 22 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 9ef2f0c8c..37813ba56 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -782,7 +782,7 @@ If the data size allocated for the sorting task is
uneven,which may lead to perf
</tr>
<tr>
<td><h5>target-file-size</h5></td>
- <td style="word-wrap: break-word;">128 mb</td>
+ <td style="word-wrap: break-word;">(none)</td>
<td>MemorySize</td>
<td>Target size of a file.</td>
</tr>
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index ccdedbd7b..b8d932e5b 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -438,7 +438,7 @@ public class CoreOptions implements Serializable {
public static final ConfigOption<MemorySize> TARGET_FILE_SIZE =
key("target-file-size")
.memoryType()
- .defaultValue(MemorySize.ofMebiBytes(128))
+ .noDefaultValue()
.withDescription("Target size of a file.");
public static final ConfigOption<Integer>
NUM_SORTED_RUNS_COMPACTION_TRIGGER =
@@ -1567,15 +1567,19 @@ public class CoreOptions implements Serializable {
return options.get(LOOKUP_CACHE_MAX_MEMORY_SIZE);
}
- public long targetFileSize() {
- return options.get(TARGET_FILE_SIZE).getBytes();
+ public long targetFileSize(TableType tableType) {
+ MemorySize memorySize = options.get(TARGET_FILE_SIZE);
+ if (memorySize == null) {
+ memorySize = tableType.getDefaultMemorySize();
+ }
+ return memorySize.getBytes();
}
- public long compactionFileSize() {
+ public long compactionFileSize(TableType tableType) {
// file size to join the compaction, we don't process on middle file
size to avoid
// compact a same file twice (the compression is not calculate so
accurately. the output
// file maybe be less than target file generated by rolling file
write).
- return options.get(TARGET_FILE_SIZE).getBytes() / 10 * 7;
+ return targetFileSize(tableType) / 10 * 7;
}
public int numSortedRunCompactionTrigger() {
@@ -2549,4 +2553,36 @@ public class CoreOptions implements Serializable {
return text(description);
}
}
+
+ /** Specifies the table type. */
+ public enum TableType implements DescribedEnum {
+ PRIMARY_KEY_TABLE(
+ "primaryKeyTable", MemorySize.ofMebiBytes(128), "The table of
primaryKey."),
+ APPEND_ONLY_TABLE(
+ "appendOnlyTable", MemorySize.ofMebiBytes(256), "The table of
appendOnly.");
+
+ private final String name;
+ private final MemorySize defaultMemorySize;
+ private final String description;
+
+ TableType(String name, MemorySize defaultMemorySize, String
description) {
+ this.name = name;
+ this.defaultMemorySize = defaultMemorySize;
+ this.description = description;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ @Override
+ public InlineElement getDescription() {
+ return text(description);
+ }
+
+ public MemorySize getDefaultMemorySize() {
+ return defaultMemorySize;
+ }
+ }
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinator.java
b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinator.java
index 614544ff1..a21b2e965 100644
---
a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinator.java
+++
b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinator.java
@@ -92,8 +92,9 @@ public class AppendOnlyTableCompactionCoordinator {
}
this.streamingMode = isStreaming;
CoreOptions coreOptions = table.coreOptions();
- this.targetFileSize = coreOptions.targetFileSize();
- this.compactionFileSize = coreOptions.compactionFileSize();
+ this.targetFileSize =
coreOptions.targetFileSize(CoreOptions.TableType.APPEND_ONLY_TABLE);
+ this.compactionFileSize =
+
coreOptions.compactionFileSize(CoreOptions.TableType.APPEND_ONLY_TABLE);
this.minFileNum = coreOptions.compactionMinFileNum();
this.maxFileNum = coreOptions.compactionMaxFileNum();
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java
b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java
index fd67c8fc9..256208338 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java
@@ -98,7 +98,7 @@ public class AppendOnlyFileStoreWrite extends
MemoryFileStoreWrite<InternalRow>
this.rowType = rowType;
this.fileFormat = options.fileFormat();
this.pathFactory = pathFactory;
- this.targetFileSize = options.targetFileSize();
+ this.targetFileSize =
options.targetFileSize(CoreOptions.TableType.APPEND_ONLY_TABLE);
this.compactionMinFileNum = options.compactionMinFileNum();
this.compactionMaxFileNum = options.compactionMaxFileNum();
this.commitForceCompact = options.commitForceCompact();
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
index d693c431b..e82ecaf1d 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
@@ -154,7 +154,7 @@ public class KeyValueFileStoreWrite extends
MemoryFileStoreWrite<KeyValue> {
valueType,
options.fileFormat(),
format2PathFactory,
- options.targetFileSize());
+
options.targetFileSize(CoreOptions.TableType.PRIMARY_KEY_TABLE));
this.keyComparatorSupplier = keyComparatorSupplier;
this.valueEqualiserSupplier = valueEqualiserSupplier;
this.mfFactory = mfFactory;
@@ -243,7 +243,7 @@ public class KeyValueFileStoreWrite extends
MemoryFileStoreWrite<KeyValue> {
levels,
compactStrategy,
keyComparator,
- options.compactionFileSize(),
+
options.compactionFileSize(CoreOptions.TableType.PRIMARY_KEY_TABLE),
options.numSortedRunStopTrigger(),
rewriter,
compactionMetrics == null
diff --git
a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinatorTest.java
b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinatorTest.java
index bd0c7da29..60446068c 100644
---
a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinatorTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinatorTest.java
@@ -69,7 +69,13 @@ public class AppendOnlyTableCompactionCoordinatorTest {
public void testMinSizeCompactTask() {
List<DataFileMeta> files =
generateNewFiles(
- 100,
appendOnlyFileStoreTable.coreOptions().targetFileSize() / 3 + 1);
+ 100,
+ appendOnlyFileStoreTable
+ .coreOptions()
+ .targetFileSize(
+
CoreOptions.TableType.APPEND_ONLY_TABLE)
+ / 3
+ + 1);
assertTasks(files, 100 / 3);
}
@@ -77,7 +83,12 @@ public class AppendOnlyTableCompactionCoordinatorTest {
public void testFilterMiddleFile() {
List<DataFileMeta> files =
generateNewFiles(
- 100,
appendOnlyFileStoreTable.coreOptions().targetFileSize() / 10 * 8);
+ 100,
+ appendOnlyFileStoreTable
+ .coreOptions()
+
.targetFileSize(CoreOptions.TableType.APPEND_ONLY_TABLE)
+ / 10
+ * 8);
assertTasks(files, 0);
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java
b/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java
index 44e2a292b..0b591b36f 100644
---
a/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java
@@ -69,7 +69,6 @@ import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
-import static org.apache.paimon.CoreOptions.TARGET_FILE_SIZE;
import static
org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory;
import static org.junit.jupiter.api.Assertions.fail;
@@ -201,7 +200,7 @@ public class ChangelogMergeTreeRewriterTest {
valueType,
new FlushingFileFormat(formatIdentifier),
Collections.singletonMap(formatIdentifier,
createNonPartFactory(path)),
- TARGET_FILE_SIZE.defaultValue().getBytes())
+
CoreOptions.TableType.PRIMARY_KEY_TABLE.getDefaultMemorySize().getBytes())
.build(BinaryRow.EMPTY_ROW, 0, new CoreOptions(new Options()));
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java
b/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java
index 0bd60c673..1c80889a4 100644
---
a/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java
@@ -62,7 +62,6 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
-import static org.apache.paimon.CoreOptions.TARGET_FILE_SIZE;
import static org.apache.paimon.io.DataFileTestUtils.row;
import static
org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory;
import static org.assertj.core.api.Assertions.assertThat;
@@ -227,7 +226,7 @@ public class ContainsLevelsTest {
rowType,
new FlushingFileFormat(identifier),
pathFactoryMap,
- TARGET_FILE_SIZE.defaultValue().getBytes())
+
CoreOptions.TableType.PRIMARY_KEY_TABLE.getDefaultMemorySize().getBytes())
.build(BinaryRow.EMPTY_ROW, 0, new CoreOptions(new Options()));
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
index b40c53d12..08b5eee17 100644
---
a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
@@ -62,7 +62,6 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
-import static org.apache.paimon.CoreOptions.TARGET_FILE_SIZE;
import static org.apache.paimon.KeyValue.UNKNOWN_SEQUENCE;
import static org.apache.paimon.io.DataFileTestUtils.row;
import static
org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory;
@@ -307,7 +306,7 @@ public class LookupLevelsTest {
rowType,
new FlushingFileFormat(identifier),
pathFactoryMap,
- TARGET_FILE_SIZE.defaultValue().getBytes())
+
CoreOptions.TableType.PRIMARY_KEY_TABLE.getDefaultMemorySize().getBytes())
.build(BinaryRow.EMPTY_ROW, 0, new CoreOptions(new Options()));
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/mergetree/MergeTreeTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/mergetree/MergeTreeTestBase.java
index 8818c1290..3b4eb017f 100644
---
a/paimon-core/src/test/java/org/apache/paimon/mergetree/MergeTreeTestBase.java
+++
b/paimon-core/src/test/java/org/apache/paimon/mergetree/MergeTreeTestBase.java
@@ -192,7 +192,7 @@ public abstract class MergeTreeTestBase {
valueType,
flushingAvro,
pathFactoryMap,
- this.options.targetFileSize());
+
this.options.targetFileSize(CoreOptions.TableType.PRIMARY_KEY_TABLE));
writerFactory = writerFactoryBuilder.build(BinaryRow.EMPTY_ROW, 0,
this.options);
compactWriterFactory = writerFactoryBuilder.build(BinaryRow.EMPTY_ROW,
0, this.options);
writer = createMergeTreeWriter(Collections.emptyList());
@@ -289,7 +289,7 @@ public abstract class MergeTreeTestBase {
options.sortedRunSizeRatio(),
options.numSortedRunCompactionTrigger()),
comparator,
- options.targetFileSize(),
+
options.targetFileSize(CoreOptions.TableType.PRIMARY_KEY_TABLE),
options.numSortedRunStopTrigger(),
new TestRewriter());
writer = createMergeTreeWriter(dataFileMetas,
mockFailResultCompactionManager);
@@ -542,7 +542,7 @@ public abstract class MergeTreeTestBase {
new Levels(comparator, files, options.numLevels()),
strategy,
comparator,
- options.compactionFileSize(),
+
options.compactionFileSize(CoreOptions.TableType.PRIMARY_KEY_TABLE),
options.numSortedRunStopTrigger(),
new TestRewriter(),
null,
