This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 278d6798061 [improve][broker] Use RoaringBitmap in tracking individual
acks to reduce memory usage (#23006)
278d6798061 is described below
commit 278d6798061c9ad82828895a2bb63d1479e6e524
Author: Lari Hotari <lhot...@users.noreply.github.com>
AuthorDate: Sat Jul 6 13:29:00 2024 +0300
[improve][broker] Use RoaringBitmap in tracking individual acks to reduce
memory usage (#23006)
(cherry picked from commit ed39c4db671c29057e51b9142a0d4cdb71e3eb88)
# Conflicts:
# distribution/server/src/assemble/LICENSE.bin.txt
---
distribution/server/src/assemble/LICENSE.bin.txt | 3 +--
distribution/shell/src/assemble/LICENSE.bin.txt | 2 ++
.../apache/bookkeeper/mledger/impl/RangeSetWrapper.java | 2 +-
pom.xml | 2 +-
pulsar-common/pom.xml | 5 +++++
.../common/util/collections/OpenLongPairRangeSet.java | 15 ++-------------
pulsar-sql/presto-distribution/LICENSE | 3 ++-
7 files changed, 14 insertions(+), 18 deletions(-)
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt
b/distribution/server/src/assemble/LICENSE.bin.txt
index 74cb4eddadd..83f9031c738 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -498,8 +498,7 @@ The Apache Software License, Version 2.0
* RxJava
- io.reactivex.rxjava3-rxjava-3.0.1.jar
* RoaringBitmap
- - org.roaringbitmap-RoaringBitmap-0.9.44.jar
- - org.roaringbitmap-shims-0.9.44.jar
+ - org.roaringbitmap-RoaringBitmap-1.2.0.jar
BSD 3-clause "New" or "Revised" License
* Google auth library
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt
b/distribution/shell/src/assemble/LICENSE.bin.txt
index 0d77d7af59f..4d35d2af3e9 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -383,6 +383,8 @@ The Apache Software License, Version 2.0
- simpleclient_tracer_common-0.16.0.jar
- simpleclient_tracer_otel-0.16.0.jar
- simpleclient_tracer_otel_agent-0.16.0.jar
+ * RoaringBitmap
+ - RoaringBitmap-1.2.0.jar
* Log4J
- log4j-api-2.18.0.jar
- log4j-core-2.18.0.jar
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java
index 299fd3dc74c..c193d71c64f 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java
@@ -55,7 +55,7 @@ public class RangeSetWrapper<T extends Comparable<T>>
implements LongPairRangeSe
this.config = managedCursor.getManagedLedger().getConfig();
this.rangeConverter = rangeConverter;
this.rangeSet = config.isUnackedRangesOpenCacheSetEnabled()
- ? new OpenLongPairRangeSet<>(4096, rangeConverter)
+ ? new OpenLongPairRangeSet<>(rangeConverter)
: new LongPairRangeSet.DefaultRangeSet<>(rangeConverter,
rangeBoundConsumer);
this.enableMultiEntry =
config.isPersistentUnackedRangesWithMultipleEntriesEnabled();
}
diff --git a/pom.xml b/pom.xml
index b5aedbc3287..e37fef6c217 100644
--- a/pom.xml
+++ b/pom.xml
@@ -308,7 +308,7 @@ flexible messaging model and an intuitive client
API.</description>
<j2objc-annotations.version>1.3</j2objc-annotations.version>
<lightproto-maven-plugin.version>0.4</lightproto-maven-plugin.version>
<dependency-check-maven.version>10.0.1</dependency-check-maven.version>
- <roaringbitmap.version>0.9.44</roaringbitmap.version>
+ <roaringbitmap.version>1.2.0</roaringbitmap.version>
<extra-enforcer-rules.version>1.6.1</extra-enforcer-rules.version>
<oshi.version>6.4.0</oshi.version>
</properties>
diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml
index 760d3dabaf7..2f11104c8ec 100644
--- a/pulsar-common/pom.xml
+++ b/pulsar-common/pom.xml
@@ -246,6 +246,11 @@
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.roaringbitmap</groupId>
+ <artifactId>RoaringBitmap</artifactId>
+ </dependency>
</dependencies>
<build>
diff --git
a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java
b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java
index c053c106be2..5114675324a 100644
---
a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java
+++
b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java
@@ -30,6 +30,7 @@ import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.commons.lang.mutable.MutableInt;
+import org.roaringbitmap.RoaringBitSet;
/**
* A Concurrent set comprising zero or more ranges of type {@link LongPair}.
This can be alternative of
@@ -46,8 +47,6 @@ import org.apache.commons.lang.mutable.MutableInt;
public class OpenLongPairRangeSet<T extends Comparable<T>> implements
LongPairRangeSet<T> {
protected final NavigableMap<Long, BitSet> rangeBitSetMap = new
ConcurrentSkipListMap<>();
- private boolean threadSafe = true;
- private final int bitSetSize;
private final LongPairConsumer<T> consumer;
// caching place-holder for cpu-optimization to avoid calculating ranges
again
@@ -57,16 +56,6 @@ public class OpenLongPairRangeSet<T extends Comparable<T>>
implements LongPairRa
private volatile boolean updatedAfterCachedForToString = true;
public OpenLongPairRangeSet(LongPairConsumer<T> consumer) {
- this(1024, true, consumer);
- }
-
- public OpenLongPairRangeSet(int size, LongPairConsumer<T> consumer) {
- this(size, true, consumer);
- }
-
- public OpenLongPairRangeSet(int size, boolean threadSafe,
LongPairConsumer<T> consumer) {
- this.threadSafe = threadSafe;
- this.bitSetSize = size;
this.consumer = consumer;
}
@@ -416,7 +405,7 @@ public class OpenLongPairRangeSet<T extends Comparable<T>>
implements LongPairRa
}
private BitSet createNewBitSet() {
- return this.threadSafe ? new ConcurrentBitSet(bitSetSize) : new
BitSet(bitSetSize);
+ return new RoaringBitSet();
}
}
diff --git a/pulsar-sql/presto-distribution/LICENSE
b/pulsar-sql/presto-distribution/LICENSE
index d18e22fcef9..98c7dda8f41 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -478,7 +478,8 @@ The Apache Software License, Version 2.0
- stream-2.9.5.jar
* High Performance Primitive Collections for Java
- hppc-0.9.1.jar
-
+ * RoaringBitmap
+ - RoaringBitmap-1.2.0.jar
Protocol Buffers License
* Protocol Buffers
