This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 3c82082806 [core] Make default of 'lookup.local-file-type' to sort
(#4622)
3c82082806 is described below
commit 3c820828062abef86b278f5a0334b6e65570c54b
Author: WenjunMin <[email protected]>
AuthorDate: Mon Dec 2 21:42:22 2024 +0800
[core] Make default of 'lookup.local-file-type' to sort (#4622)
---
docs/layouts/shortcodes/generated/core_configuration.html | 2 +-
.../org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java | 5 ++++-
.../org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java | 2 +-
paimon-common/src/main/java/org/apache/paimon/CoreOptions.java | 2 +-
.../src/main/java/org/apache/paimon/io/cache/CacheBuilder.java | 4 ++++
5 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index fad1f4907e..2ad5db28b9 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -443,7 +443,7 @@ Mainly to resolve data skew on primary keys. We recommend
starting with 64 mb wh
</tr>
<tr>
<td><h5>lookup.local-file-type</h5></td>
- <td style="word-wrap: break-word;">hash</td>
+ <td style="word-wrap: break-word;">sort</td>
<td><p>Enum</p></td>
<td>The local file type for lookup.<br /><br />Possible
values:<ul><li>"sort": Construct a sorted file for lookup.</li><li>"hash":
Construct a hash file for lookup.</li></ul></td>
</tr>
diff --git
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
index 635d876f7a..653bfee6cc 100644
---
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
+++
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
@@ -102,7 +102,10 @@ abstract class AbstractLookupBenchmark {
new CacheManager(MemorySize.ofMebiBytes(10)),
keySerializer.createSliceComparator());
- File file = new File(tempDir.toFile(), UUID.randomUUID().toString());
+ String name =
+ String.format(
+ "%s-%s-%s", options.lookupLocalFileType(),
valueLength, bloomFilterEnabled);
+ File file = new File(tempDir.toFile(), UUID.randomUUID() + "-" + name);
LookupStoreWriter writer = factory.createWriter(file,
createBloomFiler(bloomFilterEnabled));
int i = 0;
for (byte[] input : inputs) {
diff --git
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
index 9947b54a70..2d8de84327 100644
---
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
+++
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
@@ -128,7 +128,7 @@ public class LookupReaderBenchmark extends
AbstractLookupBenchmark {
LookupStoreFactory factory =
LookupStoreFactory.create(
options,
- new CacheManager(MemorySize.ofMebiBytes(10)),
+ new CacheManager(MemorySize.ofMebiBytes(20), 0.5),
new RowCompactedSerializer(RowType.of(new IntType()))
.createSliceComparator());
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index b9b5675f1d..cddef33c27 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -891,7 +891,7 @@ public class CoreOptions implements Serializable {
public static final ConfigOption<LookupLocalFileType>
LOOKUP_LOCAL_FILE_TYPE =
key("lookup.local-file-type")
.enumType(LookupLocalFileType.class)
- .defaultValue(LookupLocalFileType.HASH)
+ .defaultValue(LookupLocalFileType.SORT)
.withDescription("The local file type for lookup.");
public static final ConfigOption<Float> LOOKUP_HASH_LOAD_FACTOR =
diff --git
a/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheBuilder.java
b/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheBuilder.java
index 4660343d45..402f21f062 100644
--- a/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheBuilder.java
+++ b/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheBuilder.java
@@ -72,6 +72,10 @@ public abstract class CacheBuilder {
org.apache.paimon.shade.guava30.com.google.common.cache.CacheBuilder
.newBuilder()
.weigher(CacheBuilder::weigh)
+ // The concurrency level determines the number of
segment caches in
+ // Guava,limiting the maximum block entries held
in cache. Since we do
+ // not access this cache concurrently, it is set
to 1.
+ .concurrencyLevel(1)
.maximumWeight(memorySize.getBytes())
.removalListener(this::onRemoval)
.build());
