[
https://issues.apache.org/jira/browse/GEODE-8029?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17149663#comment-17149663
]
ASF GitHub Bot commented on GEODE-8029:
---------------------------------------
agingade commented on a change in pull request #5329:
URL: https://github.com/apache/geode/pull/5329#discussion_r448560371
##########
File path:
geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java
##########
@@ -3517,33 +3518,86 @@ public String toString() {
}
/**
- * Set of OplogEntryIds (longs). Memory is optimized by using an int[] for
ids in the unsigned int
- * range.
+ * Set of OplogEntryIds (longs).
+ * Memory is optimized by using an int[] for ids in the unsigned int range.
+ * By default we can't have more than 805306401 ids for a load factor of
0.75, the internal lists
+ * are used to overcome this limit, allowing the disk-store to recover
successfully (the internal
+ * class is **only** used during recovery to read all deleted entries).
*/
static class OplogEntryIdSet {
- private final IntOpenHashSet ints = new IntOpenHashSet((int) INVALID_ID);
- private final LongOpenHashSet longs = new LongOpenHashSet((int)
INVALID_ID);
+ private final List<IntOpenHashSet> allInts;
+ private final List<LongOpenHashSet> allLongs;
+ private final AtomicReference<IntOpenHashSet> currentInts;
+ private final AtomicReference<LongOpenHashSet> currentLongs;
+
+ // For testing purposes only.
+ @VisibleForTesting
+ OplogEntryIdSet(List<IntOpenHashSet> allInts, List<LongOpenHashSet>
allLongs) {
+ this.allInts = allInts;
+ this.currentInts = new AtomicReference<>(this.allInts.get(0));
+
+ this.allLongs = allLongs;
+ this.currentLongs = new AtomicReference<>(this.allLongs.get(0));
+ }
+
+ public OplogEntryIdSet() {
+ IntOpenHashSet intHashSet = new IntOpenHashSet((int) INVALID_ID);
+ this.allInts = new ArrayList<>();
+ this.allInts.add(intHashSet);
+ this.currentInts = new AtomicReference<>(intHashSet);
+
+ LongOpenHashSet longHashSet = new LongOpenHashSet((int) INVALID_ID);
+ this.allLongs = new ArrayList<>();
+ this.allLongs.add(longHashSet);
+ this.currentLongs = new AtomicReference<>(longHashSet);
+ }
public void add(long id) {
if (id == 0) {
throw new IllegalArgumentException();
- } else if (id > 0 && id <= 0x00000000FFFFFFFFL) {
- this.ints.add((int) id);
- } else {
- this.longs.add(id);
+ }
+
+ try {
+ if (id > 0 && id <= 0x00000000FFFFFFFFL) {
+ this.currentInts.get().add((int) id);
+ } else {
+ this.currentLongs.get().add(id);
+ }
+ } catch (IllegalArgumentException illegalArgumentException) {
+ // See GEODE-8029.
+ // Too many entries on the accumulated drf files, overflow and
continue.
+ logger.warn(
+ "There are too many entries within the disk-store, please execute
an offline compaction.",
Review comment:
"Too many entries", this is referring to deleted entries, right? In that
case how about changing it
to "large number of deleted entries"?
##########
File path:
geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java
##########
@@ -3517,33 +3518,86 @@ public String toString() {
}
/**
- * Set of OplogEntryIds (longs). Memory is optimized by using an int[] for
ids in the unsigned int
- * range.
+ * Set of OplogEntryIds (longs).
+ * Memory is optimized by using an int[] for ids in the unsigned int range.
+ * By default we can't have more than 805306401 ids for a load factor of
0.75, the internal lists
+ * are used to overcome this limit, allowing the disk-store to recover
successfully (the internal
+ * class is **only** used during recovery to read all deleted entries).
*/
static class OplogEntryIdSet {
- private final IntOpenHashSet ints = new IntOpenHashSet((int) INVALID_ID);
- private final LongOpenHashSet longs = new LongOpenHashSet((int)
INVALID_ID);
+ private final List<IntOpenHashSet> allInts;
+ private final List<LongOpenHashSet> allLongs;
+ private final AtomicReference<IntOpenHashSet> currentInts;
+ private final AtomicReference<LongOpenHashSet> currentLongs;
+
+ // For testing purposes only.
+ @VisibleForTesting
+ OplogEntryIdSet(List<IntOpenHashSet> allInts, List<LongOpenHashSet>
allLongs) {
+ this.allInts = allInts;
+ this.currentInts = new AtomicReference<>(this.allInts.get(0));
+
+ this.allLongs = allLongs;
+ this.currentLongs = new AtomicReference<>(this.allLongs.get(0));
+ }
+
+ public OplogEntryIdSet() {
+ IntOpenHashSet intHashSet = new IntOpenHashSet((int) INVALID_ID);
+ this.allInts = new ArrayList<>();
+ this.allInts.add(intHashSet);
+ this.currentInts = new AtomicReference<>(intHashSet);
+
+ LongOpenHashSet longHashSet = new LongOpenHashSet((int) INVALID_ID);
+ this.allLongs = new ArrayList<>();
+ this.allLongs.add(longHashSet);
+ this.currentLongs = new AtomicReference<>(longHashSet);
+ }
public void add(long id) {
if (id == 0) {
throw new IllegalArgumentException();
- } else if (id > 0 && id <= 0x00000000FFFFFFFFL) {
- this.ints.add((int) id);
- } else {
- this.longs.add(id);
+ }
+
+ try {
+ if (id > 0 && id <= 0x00000000FFFFFFFFL) {
+ this.currentInts.get().add((int) id);
+ } else {
+ this.currentLongs.get().add(id);
+ }
+ } catch (IllegalArgumentException illegalArgumentException) {
+ // See GEODE-8029.
+ // Too many entries on the accumulated drf files, overflow and
continue.
+ logger.warn(
+ "There are too many entries within the disk-store, please execute
an offline compaction.",
+ illegalArgumentException);
+
+ // Overflow to the next [Int|Long]OpenHashSet and continue.
+ if (id > 0 && id <= 0x00000000FFFFFFFFL) {
Review comment:
@dschneider-pivotal
Is there a scenario where the same ID is added many times? In that case,
earlier we had only one id set, and there was no chance of duplicate. Now with
the new approach, the same ID could be present in multiple id sets.
One of the caller of add is "offlineCompact()" its iterating over live
entries and calling add...Will there be two live entries with the same id?
##########
File path:
geode-core/src/main/java/org/apache/geode/internal/cache/DiskStoreImpl.java
##########
@@ -3517,33 +3518,86 @@ public String toString() {
}
/**
- * Set of OplogEntryIds (longs). Memory is optimized by using an int[] for
ids in the unsigned int
- * range.
+ * Set of OplogEntryIds (longs).
+ * Memory is optimized by using an int[] for ids in the unsigned int range.
+ * By default we can't have more than 805306401 ids for a load factor of
0.75, the internal lists
+ * are used to overcome this limit, allowing the disk-store to recover
successfully (the internal
+ * class is **only** used during recovery to read all deleted entries).
*/
static class OplogEntryIdSet {
- private final IntOpenHashSet ints = new IntOpenHashSet((int) INVALID_ID);
- private final LongOpenHashSet longs = new LongOpenHashSet((int)
INVALID_ID);
+ private final List<IntOpenHashSet> allInts;
+ private final List<LongOpenHashSet> allLongs;
+ private final AtomicReference<IntOpenHashSet> currentInts;
+ private final AtomicReference<LongOpenHashSet> currentLongs;
+
+ // For testing purposes only.
+ @VisibleForTesting
+ OplogEntryIdSet(List<IntOpenHashSet> allInts, List<LongOpenHashSet>
allLongs) {
+ this.allInts = allInts;
+ this.currentInts = new AtomicReference<>(this.allInts.get(0));
+
+ this.allLongs = allLongs;
+ this.currentLongs = new AtomicReference<>(this.allLongs.get(0));
+ }
+
+ public OplogEntryIdSet() {
+ IntOpenHashSet intHashSet = new IntOpenHashSet((int) INVALID_ID);
+ this.allInts = new ArrayList<>();
+ this.allInts.add(intHashSet);
+ this.currentInts = new AtomicReference<>(intHashSet);
+
+ LongOpenHashSet longHashSet = new LongOpenHashSet((int) INVALID_ID);
+ this.allLongs = new ArrayList<>();
+ this.allLongs.add(longHashSet);
+ this.currentLongs = new AtomicReference<>(longHashSet);
+ }
public void add(long id) {
if (id == 0) {
throw new IllegalArgumentException();
- } else if (id > 0 && id <= 0x00000000FFFFFFFFL) {
- this.ints.add((int) id);
- } else {
- this.longs.add(id);
+ }
+
+ try {
+ if (id > 0 && id <= 0x00000000FFFFFFFFL) {
+ this.currentInts.get().add((int) id);
+ } else {
+ this.currentLongs.get().add(id);
+ }
+ } catch (IllegalArgumentException illegalArgumentException) {
+ // See GEODE-8029.
+ // Too many entries on the accumulated drf files, overflow and
continue.
+ logger.warn(
+ "There are too many entries within the disk-store, please execute
an offline compaction.",
Review comment:
Do we need to pass "illegalArgumentExcpetion" to warning message? It may
not be useful for end user.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> java.lang.IllegalArgumentException: Too large (805306401 expected elements
> with load factor 0.75)
> -------------------------------------------------------------------------------------------------
>
> Key: GEODE-8029
> URL: https://issues.apache.org/jira/browse/GEODE-8029
> Project: Geode
> Issue Type: Bug
> Components: configuration, core, gfsh
> Affects Versions: 1.9.0
> Reporter: Jagadeesh sivasankaran
> Assignee: Juan Ramos
> Priority: Major
> Labels: GeodeCommons, caching-applications
> Attachments: Screen Shot 2020-04-27 at 12.21.19 PM.png, Screen Shot
> 2020-04-27 at 12.21.19 PM.png, server02.log
>
>
> we have a cluster of three Locator Geode and three Cache Server running in
> CentOS servers. Today (April 27) after patching our CENTOS servers , all
> locator and 2 servers came up , But one Cache server was not starting . here
> is the Exception details. Please let me know how to resolve the beloe issue
> and need any configuration changes to diskstore ?
>
>
> Starting a Geode Server in /app/provServerHO2...
> ....................................................................................................................................................................................................................The
> Cache Server process terminated unexpectedly with exit status 1. Please
> refer to the log file in /app/provServerHO2 for full details.
> Exception in thread "main" java.lang.IllegalArgumentException: Too large
> (805306401 expected elements with load factor 0.75)
> at it.unimi.dsi.fastutil.HashCommon.arraySize(HashCommon.java:222)
> at it.unimi.dsi.fastutil.ints.IntOpenHashSet.add(IntOpenHashSet.java:308)
> at
> org.apache.geode.internal.cache.DiskStoreImpl$OplogEntryIdSet.add(DiskStoreImpl.java:3474)
> at org.apache.geode.internal.cache.Oplog.readDelEntry(Oplog.java:3007)
> at org.apache.geode.internal.cache.Oplog.recoverDrf(Oplog.java:1500)
> at
> org.apache.geode.internal.cache.PersistentOplogSet.recoverOplogs(PersistentOplogSet.java:445)
> at
> org.apache.geode.internal.cache.PersistentOplogSet.recoverRegionsThatAreReady(PersistentOplogSet.java:369)
> at
> org.apache.geode.internal.cache.DiskStoreImpl.recoverRegionsThatAreReady(DiskStoreImpl.java:2053)
> at
> org.apache.geode.internal.cache.DiskStoreImpl.initializeIfNeeded(DiskStoreImpl.java:2041)
> security-peer-auth-init=
> at
> org.apache.geode.internal.cache.DiskStoreImpl.doInitialRecovery(DiskStoreImpl.java:2046)
> at
> org.apache.geode.internal.cache.DiskStoreFactoryImpl.initializeDiskStore(DiskStoreFactoryImpl.java:184)
> at
> org.apache.geode.internal.cache.DiskStoreFactoryImpl.create(DiskStoreFactoryImpl.java:150)
> at
> org.apache.geode.internal.cache.xmlcache.CacheCreation.createDiskStore(CacheCreation.java:794)
> at
> org.apache.geode.internal.cache.xmlcache.CacheCreation.initializePdxDiskStore(CacheCreation.java:785)
> at
> org.apache.geode.internal.cache.xmlcache.CacheCreation.create(CacheCreation.java:509)
> at
> org.apache.geode.internal.cache.xmlcache.CacheXmlParser.create(CacheXmlParser.java:337)
> at
> org.apache.geode.internal.cache.GemFireCacheImpl.loadCacheXml(GemFireCacheImpl.java:4272)
> at
> org.apache.geode.internal.cache.ClusterConfigurationLoader.applyClusterXmlConfiguration(ClusterConfigurationLoader.java:197)
> at
> org.apache.geode.internal.cache.GemFireCacheImpl.applyJarAndXmlFromClusterConfig(GemFireCacheImpl.java:1240)
> at
> org.apache.geode.internal.cache.GemFireCacheImpl.initialize(GemFireCacheImpl.java:1206)
> at
> org.apache.geode.internal.cache.InternalCacheBuilder.create(InternalCacheBuilder.java:207)
> at
> org.apache.geode.internal.cache.InternalCacheBuilder.create(InternalCacheBuilder.java:164)
> at org.apache.geode.cache.CacheFactory.create(CacheFactory.java:139)
> at
> org.apache.geode.distributed.internal.DefaultServerLauncherCacheProvider.createCache(DefaultServerLauncherCacheProvider.java:52)
> at
> org.apache.geode.distributed.ServerLauncher.createCache(ServerLauncher.java:869)
> at org.apache.geode.distributed.ServerLauncher.start(ServerLauncher.java:786)
> at org.apache.geode.distributed.ServerLauncher.run(ServerLauncher.java:716)
> at org.apache.geode.distributed.ServerLauncher.main(ServerLauncher.java:236)
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
