(cassandra-accord) branch cep-15-accord-stabilize-tests updated (78badf67 -> 99f9f611)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cep-15-accord-stabilize-tests in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git from 78badf67 Pull in Ariel's fix new 080e4173 Revert "Pull in Ariel's fix" new a8decd72 Fix timeout deadlock new 99f9f611 Fix compile The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/accord/impl/DefaultRequestTimeouts.java | 49 +++--- .../src/main/java/accord/utils/ArrayBuffers.java | 44 +-- 2 files changed, 75 insertions(+), 18 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 02/03: Fix timeout deadlock
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord-stabilize-tests
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit a8decd72ae9cc401cf4166000586be7752ba144a
Author: Benedict Elliott Smith
AuthorDate: Tue Oct 1 10:34:42 2024 +0100
Fix timeout deadlock
---
.../java/accord/impl/DefaultRequestTimeouts.java | 45 +-
.../src/main/java/accord/utils/ArrayBuffers.java | 37 ++
2 files changed, 73 insertions(+), 9 deletions(-)
diff --git a/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
b/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
index 8f11e5f1..d278eafd 100644
--- a/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
+++ b/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
@@ -24,6 +24,8 @@ import java.util.function.Function;
import accord.api.RequestTimeouts;
import accord.local.Node;
+import accord.utils.ArrayBuffers;
+import accord.utils.ArrayBuffers.BufferList;
import accord.utils.LogGroupTimers;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -91,16 +93,41 @@ public class DefaultRequestTimeouts implements
RequestTimeouts
@Override
public void run()
{
-lock.lock();
-try
+try (BufferList collect = new BufferList<>())
{
-long now = node.elapsed(MILLISECONDS);
-// TODO (expected): should we handle reentrancy? Or at least
throw an exception?
-timeouts.advance(now, this, (s, r) -> r.timeout.timeout());
-}
-finally
-{
-lock.unlock();
+int i = 0;
+try
+{
+lock.lock();
+try
+{
+long now = node.elapsed(MILLISECONDS);
+// TODO (expected): should we handle reentrancy? Or at
least throw an exception?
+timeouts.advance(now, collect, BufferList::add);
+}
+finally
+{
+lock.unlock();
+}
+
+while (i < collect.size())
+collect.get(i++).timeout.timeout();
+}
+catch (Throwable t)
+{
+while (i < collect.size())
+{
+try
+{
+collect.get(i++).timeout.timeout();
+}
+catch (Throwable t2)
+{
+t.addSuppressed(t2);
+}
+}
+throw t;
+}
}
}
diff --git a/accord-core/src/main/java/accord/utils/ArrayBuffers.java
b/accord-core/src/main/java/accord/utils/ArrayBuffers.java
index 388c5e2c..5973c37b 100644
--- a/accord-core/src/main/java/accord/utils/ArrayBuffers.java
+++ b/accord-core/src/main/java/accord/utils/ArrayBuffers.java
@@ -23,8 +23,11 @@ import accord.api.RoutingKey;
import accord.primitives.Range;
import accord.primitives.TxnId;
+import java.io.Closeable;
import java.lang.reflect.Array;
+import java.util.AbstractList;
import java.util.Arrays;
+import java.util.List;
import java.util.function.IntFunction;
import static accord.utils.Invariants.illegalState;
@@ -783,4 +786,38 @@ public class ArrayBuffers
}
}
+public static class BufferList extends AbstractList implements
Closeable
+{
+private static final Object[] EMPTY = new Object[0];
+private Object[] buffer = EMPTY;
+private int size;
+
+@Override
+public E get(int index)
+{
+return (E) buffer[index];
+}
+
+@Override
+public int size()
+{
+return size;
+}
+
+@Override
+public boolean add(E e)
+{
+if (size == buffer.length)
+buffer = cachedAny().resize(buffer, size, Math.min(8, size *
2));
+buffer[size++] = e;
+return true;
+}
+
+public void close()
+{
+if (buffer == null) return;
+cachedAny().forceDiscard(buffer, size);
+buffer = null;
+}
+}
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 03/03: Fix compile
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch cep-15-accord-stabilize-tests in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git commit 99f9f61182eeae916c3dba9ab6b02fb647b581bd Author: Alex Petrov AuthorDate: Tue Oct 1 12:07:17 2024 +0200 Fix compile --- .../src/main/java/accord/impl/DefaultRequestTimeouts.java | 1 - accord-core/src/main/java/accord/utils/ArrayBuffers.java | 11 +-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java b/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java index d278eafd..a9ce7eb2 100644 --- a/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java +++ b/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java @@ -24,7 +24,6 @@ import java.util.function.Function; import accord.api.RequestTimeouts; import accord.local.Node; -import accord.utils.ArrayBuffers; import accord.utils.ArrayBuffers.BufferList; import accord.utils.LogGroupTimers; diff --git a/accord-core/src/main/java/accord/utils/ArrayBuffers.java b/accord-core/src/main/java/accord/utils/ArrayBuffers.java index 5973c37b..3be78b3f 100644 --- a/accord-core/src/main/java/accord/utils/ArrayBuffers.java +++ b/accord-core/src/main/java/accord/utils/ArrayBuffers.java @@ -18,18 +18,17 @@ package accord.utils; -import accord.api.Key; -import accord.api.RoutingKey; -import accord.primitives.Range; -import accord.primitives.TxnId; - import java.io.Closeable; import java.lang.reflect.Array; import java.util.AbstractList; import java.util.Arrays; -import java.util.List; import java.util.function.IntFunction; +import accord.api.Key; +import accord.api.RoutingKey; +import accord.primitives.Range; +import accord.primitives.TxnId; + import static accord.utils.Invariants.illegalState; /** - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 01/03: Revert "Pull in Ariel's fix"
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord-stabilize-tests
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 080e4173bcb3f12fe47e5b1d8acefbae4187e0c6
Author: Alex Petrov
AuthorDate: Tue Oct 1 11:44:15 2024 +0200
Revert "Pull in Ariel's fix"
This reverts commit 78badf67faf811ae5ee3e709e28f9b52223784ec.
---
accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java | 7 +--
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
b/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
index e9f8c817..8f11e5f1 100644
--- a/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
+++ b/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
@@ -18,8 +18,6 @@
package accord.impl;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;
@@ -93,19 +91,16 @@ public class DefaultRequestTimeouts implements
RequestTimeouts
@Override
public void run()
{
-List timedOut = new ArrayList<>();
lock.lock();
try
{
long now = node.elapsed(MILLISECONDS);
// TODO (expected): should we handle reentrancy? Or at least
throw an exception?
-timeouts.advance(now, this, (s, r) -> timedOut.add(r.timeout));
+timeouts.advance(now, this, (s, r) -> r.timeout.timeout());
}
finally
{
lock.unlock();
-for (Timeout timeout : timedOut)
-timeout.timeout();
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 01/01: Pull in Ariel's fix
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord-stabilize-tests
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 78badf67faf811ae5ee3e709e28f9b52223784ec
Author: Alex Petrov
AuthorDate: Tue Oct 1 10:25:54 2024 +0200
Pull in Ariel's fix
---
accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
b/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
index 8f11e5f1..e9f8c817 100644
--- a/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
+++ b/accord-core/src/main/java/accord/impl/DefaultRequestTimeouts.java
@@ -18,6 +18,8 @@
package accord.impl;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;
@@ -91,16 +93,19 @@ public class DefaultRequestTimeouts implements
RequestTimeouts
@Override
public void run()
{
+List timedOut = new ArrayList<>();
lock.lock();
try
{
long now = node.elapsed(MILLISECONDS);
// TODO (expected): should we handle reentrancy? Or at least
throw an exception?
-timeouts.advance(now, this, (s, r) -> r.timeout.timeout());
+timeouts.advance(now, this, (s, r) -> timedOut.add(r.timeout));
}
finally
{
lock.unlock();
+for (Timeout timeout : timedOut)
+timeout.timeout();
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch cep-15-accord-stabilize-tests created (now 78badf67)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cep-15-accord-stabilize-tests in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git at 78badf67 Pull in Ariel's fix This branch includes the following new commits: new 78badf67 Pull in Ariel's fix The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cep-15-accord updated: Add purging to Accord Journal table
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git The following commit(s) were added to refs/heads/cep-15-accord by this push: new 4eb540305d Add purging to Accord Journal table 4eb540305d is described below commit 4eb540305da63766d43a40c178dba3505df6d4cd Author: Alex Petrov AuthorDate: Mon Sep 30 15:30:28 2024 +0200 Add purging to Accord Journal table Patch by Alex Petrov; reviewed by Aleksey Yeshchenko and Benedict Elliott Smith for CASSANDRA-19877 --- .../db/compaction/CompactionIterator.java | 178 - .../cassandra/service/accord/AccordJournal.java| 8 +- .../cassandra/service/accord/AccordKeyspace.java | 2 +- .../cassandra/service/accord/AccordService.java| 77 + .../cassandra/service/accord/IAccordService.java | 3 + .../cassandra/service/accord/JournalKey.java | 6 +- .../cassandra/service/accord/SavedCommand.java | 74 + .../accord/AccordJournalCompactionTest.java| 29 ++-- 8 files changed, 323 insertions(+), 54 deletions(-) diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java index 57bf16ba15..17160c4fa8 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.db.compaction; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; @@ -32,6 +33,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Ordering; import accord.local.Cleanup; +import accord.local.CommandStores; import accord.local.CommandStores.RangesForEpoch; import accord.local.DurableBefore; import accord.local.RedundantBefore; @@ -43,6 +45,7 @@ import accord.primitives.Timestamp; import accord.primitives.TxnId; import org.agrona.collections.Int2ObjectHashMap; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.db.AbstractCompactionController; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Columns; @@ -53,6 +56,8 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.partitions.PurgeFunction; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; @@ -73,12 +78,18 @@ import org.apache.cassandra.index.transactions.CompactionTransaction; import org.apache.cassandra.index.transactions.IndexTransaction; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.journal.KeySupport; import org.apache.cassandra.metrics.TopPartitionTracker; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.accord.AccordJournalValueSerializers; +import org.apache.cassandra.service.accord.AccordJournalValueSerializers.FlyweightSerializer; import org.apache.cassandra.service.accord.AccordKeyspace; import org.apache.cassandra.service.accord.AccordKeyspace.CommandRows; import org.apache.cassandra.service.accord.AccordKeyspace.CommandsColumns; @@ -86,12 +97,15 @@ import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor import org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyRows; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.IAccordService; +import org.apache.cassandra.service.accord.JournalKey; +import org.apache.cassandra.service.accord.SavedCommand; import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey; import org.apache.cassandra.service.paxos.PaxosRepairHistory; import org.apache.cassandra.service.paxos.uncommitted.PaxosRows; import org.apache.cassandra.utils.TimeUUID; import static accord.local.Cleanup.TRUNCATE_WITH_OUTCOME; +import static accord.local.Cleanup.shouldCleanup; import static accord.local.Cleanup.shouldCleanupPartial; import static
(cassandra) branch cep-15-accord updated: Ninja: Repin accord
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git The following commit(s) were added to refs/heads/cep-15-accord by this push: new 0b589d0ee5 Ninja: Repin accord 0b589d0ee5 is described below commit 0b589d0ee5e91eb8f084bc7cfbebe774aee57c5a Author: Alex Petrov AuthorDate: Mon Sep 30 23:03:06 2024 +0200 Ninja: Repin accord --- modules/accord | 2 +- .../org/apache/cassandra/service/accord/AccordConfigurationService.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/accord b/modules/accord index 4a39a2d0d2..b0ad8e8cfb 16 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 4a39a2d0d286f6b03538ce44fbf7c9cbc1b1cf48 +Subproject commit b0ad8e8cfbcc35fc8152850760cd1f34f06836e7 diff --git a/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java b/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java index 74e1d6052a..09a04140c9 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java @@ -60,7 +60,6 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Simulate; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Future; -import org.apache.cassandra.utils.concurrent.Threads; import static org.apache.cassandra.utils.Simulate.With.MONITORS; - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 05/09: refine Cleanup logic to support C* cleanup
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch ex-sp-pl
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 1a9b54b22199f8828b4eaaa4c4255d3ecc76a868
Author: Benedict Elliott Smith
AuthorDate: Sun Sep 29 11:15:55 2024 +0100
refine Cleanup logic to support C* cleanup
---
.../src/main/java/accord/local/Cleanup.java| 92 ++
.../src/main/java/accord/local/Commands.java | 1 +
.../main/java/accord/local/RedundantBefore.java| 5 ++
.../main/java/accord/utils/ReducingRangeMap.java | 18 -
4 files changed, 100 insertions(+), 16 deletions(-)
diff --git a/accord-core/src/main/java/accord/local/Cleanup.java
b/accord-core/src/main/java/accord/local/Cleanup.java
index 888fe0ba..765fddeb 100644
--- a/accord-core/src/main/java/accord/local/Cleanup.java
+++ b/accord-core/src/main/java/accord/local/Cleanup.java
@@ -19,7 +19,11 @@
package accord.local;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import accord.api.VisibleForImplementation;
+import accord.primitives.FullRoute;
+import accord.primitives.Route;
import accord.primitives.SaveStatus;
import accord.primitives.Status.Durability;
import accord.primitives.TxnId;
@@ -52,7 +56,9 @@ public enum Cleanup
TRUNCATE(TruncatedApply),
INVALIDATE(Invalidated),
VESTIGIAL(ErasedOrVestigial),
-ERASE(Erased);
+ERASE(Erased),
+// we can stop storing the record entirely
+EXPUNGE(Erased);
public final SaveStatus appliesIfNot;
@@ -84,20 +90,24 @@ public enum Cleanup
public static Cleanup shouldCleanup(TxnId txnId, SaveStatus status,
Durability durability, StoreParticipants participants, RedundantBefore
redundantBefore, DurableBefore durableBefore)
{
-return shouldCleanup(txnId, status, durability, participants,
redundantBefore, durableBefore, true);
+return shouldCleanupInternal(txnId, status, durability, participants,
redundantBefore, durableBefore).filter(status);
}
-public static Cleanup shouldCleanup(TxnId txnId, SaveStatus status,
Durability durability, StoreParticipants participants, RedundantBefore
redundantBefore, DurableBefore durableBefore, boolean enforceInvariants)
+@VisibleForImplementation
+public static Cleanup shouldCleanupPartial(TxnId txnId, SaveStatus status,
Durability durability, StoreParticipants participants, RedundantBefore
redundantBefore, DurableBefore durableBefore)
{
-return shouldCleanupInternal(txnId, status, durability, participants,
redundantBefore, durableBefore, enforceInvariants).filter(status);
+return shouldCleanupPartialInternal(txnId, status, durability,
participants, redundantBefore, durableBefore).filter(status);
}
-private static Cleanup shouldCleanupInternal(TxnId txnId, SaveStatus
status, Durability durability, StoreParticipants participants, RedundantBefore
redundantBefore, DurableBefore durableBefore, boolean enforceInvariants)
+private static Cleanup shouldCleanupInternal(TxnId txnId, SaveStatus
saveStatus, Durability durability, StoreParticipants participants,
RedundantBefore redundantBefore, DurableBefore durableBefore)
{
if (txnId.kind() == EphemeralRead)
return NO; // TODO (required): clean-up based on timeout
-if (status == Uninitialised)
+if (expunge(txnId, saveStatus, durableBefore, redundantBefore))
+return EXPUNGE;
+
+if (saveStatus == Uninitialised)
{
if (!redundantBefore.isAnyOnAnyEpoch(txnId, participants.touches,
SHARD_REDUNDANT))
return NO;
@@ -116,22 +126,60 @@ public enum Cleanup
return cleanup;
}
-if (!status.hasBeen(PreCommitted) &&
redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns,
SHARD_REDUNDANT))
-return Cleanup.INVALIDATE;
-
if (!participants.hasFullRoute())
{
-if (status == Invalidated && durableBefore.min(txnId) ==
UniversalOrInvalidated)
-return Cleanup.ERASE;
+if (!saveStatus.hasBeen(PreCommitted) &&
redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns,
SHARD_REDUNDANT))
+return Cleanup.INVALIDATE;
return Cleanup.NO;
}
+return cleanupWithFullRoute(false, participants, txnId, saveStatus,
durability, redundantBefore, durableBefore);
+}
+
+private static Cleanup shouldCleanupPartialInternal(TxnId txnId,
SaveStatus status, @Nullable Durability durability, @Nullable StoreParticipants
participants, RedundantBefore redundantBefore, DurableBefore durableBefore)
+{
+if (txnId.kind() == EphemeralRead)
+return NO; // TODO (required): clean-up based on timeout
+
+if (expunge(txnId, status, durableBefore, redundantBefore))
+return EXPUNGE;
+
+
(cassandra-accord) 06/09: EXPUNGE_PARTIAL
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch ex-sp-pl
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 97f73d926ba904fa071e2fdfe8e61c6ebeb6dc31
Author: Benedict Elliott Smith
AuthorDate: Sun Sep 29 11:31:40 2024 +0100
EXPUNGE_PARTIAL
---
accord-core/src/main/java/accord/local/Cleanup.java| 5 -
accord-core/src/main/java/accord/local/cfk/CommandsForKey.java | 2 +-
accord-core/src/main/java/accord/local/cfk/Updating.java | 9 +++--
3 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/accord-core/src/main/java/accord/local/Cleanup.java
b/accord-core/src/main/java/accord/local/Cleanup.java
index 765fddeb..63ef7664 100644
--- a/accord-core/src/main/java/accord/local/Cleanup.java
+++ b/accord-core/src/main/java/accord/local/Cleanup.java
@@ -57,6 +57,8 @@ public enum Cleanup
INVALIDATE(Invalidated),
VESTIGIAL(ErasedOrVestigial),
ERASE(Erased),
+// erase all fields except any participants and committed executeAt
+EXPUNGE_PARTIAL(Erased),
// we can stop storing the record entirely
EXPUNGE(Erased);
@@ -165,7 +167,7 @@ public enum Cleanup
case Applying:
return TRUNCATE_WITH_OUTCOME;
default:
-return EXPUNGE;
+return EXPUNGE_PARTIAL;
}
}
@@ -217,6 +219,7 @@ public enum Cleanup
case UniversalOrInvalidated:
case Universal:
+// TODO (expected): can we EXPUNGE here?
return Cleanup.ERASE;
}
}
diff --git a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
index 5aad9df6..b1489ac0 100644
--- a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
+++ b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
@@ -1346,7 +1346,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
implements CommandsSumm
TxnInfo txn = byId[i];
if (txn.status() == INVALID_OR_TRUNCATED_OR_PRUNED) continue;
if (txn.isCommittedAndExecutes()) ++countCommitted;
-else if (minUndecidedById == -1 && mayExecute(newBoundsInfo, txn)
&& isPostBootstrapAndOwned(txn, newBoundsInfo))
+else if (minUndecidedById == -1 && !txn.isCommittedToExecute() &&
txn.mayExecute())
minUndecidedById = i;
}
TxnInfo[] committedByExecuteAt = new TxnInfo[countCommitted];
diff --git a/accord-core/src/main/java/accord/local/cfk/Updating.java
b/accord-core/src/main/java/accord/local/cfk/Updating.java
index 86b40e94..1e8b319a 100644
--- a/accord-core/src/main/java/accord/local/cfk/Updating.java
+++ b/accord-core/src/main/java/accord/local/cfk/Updating.java
@@ -614,7 +614,7 @@ class Updating
int maxAppliedWriteByExecuteAt =
cfk.maxAppliedWriteByExecuteAt;
if (pos <= maxAppliedWriteByExecuteAt)
{
-if (pos < maxAppliedWriteByExecuteAt && !wasPruned &&
cfk.isPostBootstrapAndOwned(newInfo))
+if (pos < maxAppliedWriteByExecuteAt && !wasPruned)
{
for (int i = pos; i <= maxAppliedWriteByExecuteAt; ++i)
{
@@ -671,16 +671,13 @@ class Updating
private static int
maybeAdvanceMaxAppliedAndCheckForLinearizabilityViolations(CommandsForKey cfk,
int appliedPos, Txn.Kind appliedKind, TxnInfo applied, boolean wasPruned)
{
-if (!wasPruned && cfk.isPostBootstrapAndOwned(applied))
+if (!wasPruned)
{
TxnInfo[] committedByExecuteAt = cfk.committedByExecuteAt;
for (int i = cfk.maxAppliedWriteByExecuteAt + 1; i < appliedPos ;
++i)
{
if (committedByExecuteAt[i].status() != APPLIED
&& appliedKind.witnesses(committedByExecuteAt[i])
-&& cfk.isPostBootstrapAndOwned(committedByExecuteAt[i])
-&& cfk.boundsInfo.startOwnershipEpoch <=
applied.executeAt.epoch()
-&& cfk.boundsInfo.endOwnershipEpoch >
applied.executeAt.epoch()
&& reportLinearizabilityViolations())
logger.error("Linearizability violation on key {}: {}
is committed to execute (at {}) before {} that should witness it but has
already applied (at {})", cfk.key, committedByExecuteAt[i].plainTxnId(),
committedByExecuteAt[i].plainExecuteAt(), applied.plainTxnId(),
applied.plainExecuteAt());
}
@@ -696,7 +693,7 @@ class Updating
if (pos == infos.length)
return -1;
-if (infos[pos].status().compareTo(COMMITTED) <
(cassandra-accord) branch ex-sp-pl created (now fbc941ca)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch ex-sp-pl in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git at fbc941ca Fix build This branch includes the following new commits: new 34c35d8c squash 19869 to 80414c2e new 4eddc493 Revisit eppch interval handling and inference of invalidations new 1999a635 unused imports new f5b25b5b fix replay new 1a9b54b2 refine Cleanup logic to support C* cleanup new 97f73d92 EXPUNGE_PARTIAL new a396b993 fix replay new 65caf7fc fix replay new fbc941ca Fix build The 9 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 07/09: fix replay
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch ex-sp-pl
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit a396b99393ed19b3995d0afbccf9340a3c413614
Author: Benedict Elliott Smith
AuthorDate: Sun Sep 29 11:57:06 2024 +0100
fix replay
---
.../src/test/java/accord/impl/list/ListStore.java | 18 +-
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/accord-core/src/test/java/accord/impl/list/ListStore.java
b/accord-core/src/test/java/accord/impl/list/ListStore.java
index d2a071bd..3671e3b6 100644
--- a/accord-core/src/test/java/accord/impl/list/ListStore.java
+++ b/accord-core/src/test/java/accord/impl/list/ListStore.java
@@ -233,15 +233,15 @@ public class ListStore implements DataStore
public void restoreFromSnapshot()
{
-if (snapshot == null)
-return;
-
-data.putAll(snapshot.data);
-addedAts.addAll(snapshot.addedAts);
-removedAts.addAll(snapshot.removedAts);
-purgedAts.addAll(snapshot.purgedAts);
-fetchCompletes.addAll(snapshot.fetchCompletes);
-pendingRemoves.addAll(snapshot.pendingRemoves);
+if (snapshot != null)
+{
+data.putAll(snapshot.data);
+addedAts.addAll(snapshot.addedAts);
+removedAts.addAll(snapshot.removedAts);
+purgedAts.addAll(snapshot.purgedAts);
+fetchCompletes.addAll(snapshot.fetchCompletes);
+pendingRemoves.addAll(snapshot.pendingRemoves);
+}
while (!pendingSnapshots.isEmpty())
pendingSnapshots.pollFirst().onCompletion.accept(false);
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 03/09: unused imports
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch ex-sp-pl in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git commit 1999a635c36739df37c67f27222d66b9c5c040d4 Author: Benedict Elliott Smith AuthorDate: Sun Sep 29 08:41:06 2024 +0100 unused imports --- accord-core/src/main/java/accord/api/Agent.java| 3 +- .../java/accord/impl/InMemoryCommandStore.java | 4 -- .../src/main/java/accord/local/Cleanup.java| 1 - .../src/main/java/accord/local/CommandStore.java | 3 +- .../src/main/java/accord/local/Commands.java | 1 - .../main/java/accord/local/RedundantBefore.java| 45 ++ .../main/java/accord/local/SafeCommandStore.java | 1 - .../main/java/accord/local/StoreParticipants.java | 2 +- .../main/java/accord/local/cfk/CommandsForKey.java | 14 --- .../main/java/accord/local/cfk/PostProcess.java| 2 - .../src/main/java/accord/topology/Topologies.java | 6 --- .../src/main/java/accord/topology/Topology.java| 1 - .../src/test/java/accord/impl/TestAgent.java | 4 +- .../src/test/java/accord/impl/basic/Cluster.java | 44 +++-- .../accord/impl/basic/DelayedCommandStores.java| 6 +++ .../src/test/java/accord/impl/basic/Journal.java | 7 ++-- .../src/test/java/accord/impl/list/ListAgent.java | 2 +- .../src/test/java/accord/impl/list/ListStore.java | 26 + .../java/accord/local/cfk/CommandsForKeyTest.java | 4 +- .../main/java/accord/maelstrom/MaelstromAgent.java | 2 +- 20 files changed, 90 insertions(+), 88 deletions(-) diff --git a/accord-core/src/main/java/accord/api/Agent.java b/accord-core/src/main/java/accord/api/Agent.java index 2f42da1d..38ff4ccd 100644 --- a/accord-core/src/main/java/accord/api/Agent.java +++ b/accord-core/src/main/java/accord/api/Agent.java @@ -64,8 +64,7 @@ public interface Agent extends UncaughtExceptionListener @Override void onUncaughtException(Throwable t); - -void onHandledException(Throwable t); +void onHandledException(Throwable t, String context); /** * @return PreAccept timeout with implementation-defined resolution of the hybrid logical clock diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java index 5b425201..8f32380e 100644 --- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java +++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java @@ -53,7 +53,6 @@ import org.slf4j.LoggerFactory; import accord.api.Agent; import accord.api.DataStore; -import accord.api.Key; import accord.api.ProgressLog; import accord.local.Cleanup; import accord.local.Command; @@ -72,7 +71,6 @@ import accord.local.cfk.CommandsForKey; import accord.primitives.AbstractRanges; import accord.primitives.AbstractUnseekableKeys; import accord.primitives.Deps; -import accord.primitives.Keys; import accord.primitives.PartialDeps; import accord.primitives.Participants; import accord.primitives.Range; @@ -81,7 +79,6 @@ import accord.primitives.Routable.Domain; import accord.primitives.RoutableKey; import accord.primitives.Routables; import accord.primitives.Route; -import accord.primitives.RoutingKeys; import accord.primitives.SaveStatus; import accord.primitives.Status; import accord.primitives.Timestamp; @@ -90,7 +87,6 @@ import accord.primitives.TxnId; import accord.primitives.Unseekable; import accord.primitives.Unseekables; import accord.utils.Invariants; -import accord.utils.ReducingRangeMap; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; diff --git a/accord-core/src/main/java/accord/local/Cleanup.java b/accord-core/src/main/java/accord/local/Cleanup.java index b1a4a901..888fe0ba 100644 --- a/accord-core/src/main/java/accord/local/Cleanup.java +++ b/accord-core/src/main/java/accord/local/Cleanup.java @@ -23,7 +23,6 @@ import javax.annotation.Nonnull; import accord.primitives.SaveStatus; import accord.primitives.Status.Durability; import accord.primitives.TxnId; -import accord.primitives.Unseekables; import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY; import static accord.local.RedundantStatus.LIVE; diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java index e149f1ad..19db3e7a 100644 --- a/accord-core/src/main/java/accord/local/CommandStore.java +++ b/accord-core/src/main/java/accord/local/CommandStore.java @@ -161,7 +161,6 @@ public abstract class CommandStore implements AgentExecutor private DurableBefore durableBefore = DurableBefore.EMPTY; private MaxConflicts maxConflicts = MaxConflicts.EMPTY; protected RangesForEpoch rangesForEpoch; -protected Ranges activeRanges, leavingRanges; /** * safeToRead is related to RedundantBefore, but a distinct concept. @@ -535,7 +534,7
(cassandra-accord) 04/09: fix replay
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch ex-sp-pl
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit f5b25b5b1d2858a66e45aba409a3d2a201b8f6a2
Author: Benedict Elliott Smith
AuthorDate: Sun Sep 29 09:15:52 2024 +0100
fix replay
---
accord-core/src/main/java/accord/impl/InMemoryCommandStore.java | 3 ++-
accord-core/src/test/java/accord/impl/basic/Journal.java| 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index 8f32380e..89fe806f 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -101,6 +101,7 @@ import static accord.primitives.SaveStatus.ReadyToExecute;
import static accord.primitives.Status.Applied;
import static accord.primitives.Status.Durability.Local;
import static accord.primitives.Status.Invalidated;
+import static accord.primitives.Status.PreApplied;
import static accord.primitives.Status.PreCommitted;
import static accord.primitives.Status.Stable;
import static accord.primitives.Status.Truncated;
@@ -1397,7 +1398,7 @@ public abstract class InMemoryCommandStore extends
CommandStore
safeStore -> {
SafeCommand safeCommand =
safeStore.unsafeGet(txnId);
Command local = safeCommand.current();
- if (local.is(Stable) &&
!local.hasBeen(Applied))
+ if (local.is(Stable) ||
local.is(PreApplied))
Commands.maybeExecute(safeStore,
safeCommand, local, true, true);
else if
(local.saveStatus().compareTo(Applying) >= 0 && !local.is(Invalidated) &&
!local.is(Truncated))
Commands.applyWrites(safeStore,
context, local).begin(agent);
diff --git a/accord-core/src/test/java/accord/impl/basic/Journal.java
b/accord-core/src/test/java/accord/impl/basic/Journal.java
index 019801b7..1a53472b 100644
--- a/accord-core/src/test/java/accord/impl/basic/Journal.java
+++ b/accord-core/src/test/java/accord/impl/basic/Journal.java
@@ -55,6 +55,7 @@ import accord.primitives.Writes;
import accord.utils.Invariants;
import org.agrona.collections.Long2ObjectHashMap;
+import static accord.primitives.SaveStatus.Stable;
import static accord.primitives.Status.Invalidated;
import static accord.primitives.Status.Truncated;
import static accord.utils.Invariants.illegalState;
@@ -134,7 +135,7 @@ public class Journal
for (TxnId txnId : diffs.keySet())
{
Command command = reconstruct(commandStoreId, txnId);
-if (command.saveStatus().compareTo(SaveStatus.Applying) >= 0
&& !command.is(Invalidated) && !command.is(Truncated))
+if (command.saveStatus().compareTo(Stable) >= 0 &&
!command.hasBeen(Truncated))
toApply.add(command);
loader.load(command);
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 09/09: Fix build
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch ex-sp-pl
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit fbc941ca7a6041732174b9d8011a29e1e138d7e7
Author: Alex Petrov
AuthorDate: Sun Sep 29 18:25:29 2024 +0200
Fix build
---
accord-core/src/main/java/accord/impl/InMemoryCommandStore.java | 7 ++-
accord-core/src/main/java/accord/local/cfk/CommandsForKey.java | 2 +-
2 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index 11eaf5f9..b203575e 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -1404,7 +1404,7 @@ public abstract class InMemoryCommandStore extends
CommandStore
}
else if
(local.saveStatus().compareTo(Applying) >= 0 && !local.hasBeen(Truncated))
{
- unsafeApplyWrites(safeStore, context,
safeCommand, local);
+ unsafeApplyWrites(safeStore,
safeCommand, local);
}
return null;
});
@@ -1412,9 +1412,8 @@ public abstract class InMemoryCommandStore extends
CommandStore
};
}
-public static void unsafeApplyWrites(SafeCommandStore safeStore,
PreLoadContext context, SafeCommand safeCommand, Command command)
+public static void unsafeApplyWrites(SafeCommandStore safeStore,
SafeCommand safeCommand, Command command)
{
-CommandStore unsafeStore = safeStore.commandStore();
Command.Executed executed = command.asExecuted();
Participants executes = executed.participants().executes(safeStore,
command.txnId(), command.executeAt());
if (!executes.isEmpty())
@@ -1425,8 +1424,6 @@ public abstract class InMemoryCommandStore extends
CommandStore
}
}
-
-
@VisibleForTesting
public void load(Deps loading)
{
diff --git a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
index b1489ac0..95fe7950 100644
--- a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
+++ b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
@@ -775,7 +775,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
implements CommandsSumm
return this == COMMITTED | this == STABLE | this == APPLIED;
}
-Timestamp depsKnownBefore(TxnId txnId, Timestamp executeAt)
+public Timestamp depsKnownBefore(TxnId txnId, Timestamp executeAt)
{
return depsKnownUntilExecuteAt() ? executeAt : txnId;
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 08/09: fix replay
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch ex-sp-pl
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 65caf7fc31bc1559457cff63ec2b3f6089930f26
Author: Benedict Elliott Smith
AuthorDate: Sun Sep 29 15:53:16 2024 +0100
fix replay
---
accord-core/src/main/java/accord/api/Write.java| 2 ++
.../java/accord/impl/InMemoryCommandStore.java | 23 --
.../src/main/java/accord/local/Cleanup.java| 1 +
.../src/main/java/accord/local/Commands.java | 17 +---
.../src/main/java/accord/primitives/Writes.java| 11 +++
.../src/test/java/accord/impl/list/ListWrite.java | 12 +++
6 files changed, 48 insertions(+), 18 deletions(-)
diff --git a/accord-core/src/main/java/accord/api/Write.java
b/accord-core/src/main/java/accord/api/Write.java
index 7f3fad7d..e6052fdb 100644
--- a/accord-core/src/main/java/accord/api/Write.java
+++ b/accord-core/src/main/java/accord/api/Write.java
@@ -33,4 +33,6 @@ import accord.utils.async.AsyncChain;
public interface Write
{
AsyncChain apply(Seekable key, SafeCommandStore safeStore, TxnId
txnId, Timestamp executeAt, DataStore store, PartialTxn txn);
+// TODO (expected): this is used only for testing today; hide it somewhere
outside of the public API
+default void applyUnsafe(Seekable key, SafeCommandStore safeStore, TxnId
txnId, Timestamp executeAt, DataStore store, PartialTxn txn) { throw new
UnsupportedOperationException(); }
}
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index 89fe806f..11eaf5f9 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -1399,15 +1399,34 @@ public abstract class InMemoryCommandStore extends
CommandStore
SafeCommand safeCommand =
safeStore.unsafeGet(txnId);
Command local = safeCommand.current();
if (local.is(Stable) ||
local.is(PreApplied))
+ {
Commands.maybeExecute(safeStore,
safeCommand, local, true, true);
- else if
(local.saveStatus().compareTo(Applying) >= 0 && !local.is(Invalidated) &&
!local.is(Truncated))
- Commands.applyWrites(safeStore,
context, local).begin(agent);
+ }
+ else if
(local.saveStatus().compareTo(Applying) >= 0 && !local.hasBeen(Truncated))
+ {
+ unsafeApplyWrites(safeStore, context,
safeCommand, local);
+ }
return null;
});
}
};
}
+public static void unsafeApplyWrites(SafeCommandStore safeStore,
PreLoadContext context, SafeCommand safeCommand, Command command)
+{
+CommandStore unsafeStore = safeStore.commandStore();
+Command.Executed executed = command.asExecuted();
+Participants executes = executed.participants().executes(safeStore,
command.txnId(), command.executeAt());
+if (!executes.isEmpty())
+{
+command.writes().applyUnsafe(safeStore,
Commands.applyRanges(safeStore, command.executeAt()), command.partialTxn());
+safeCommand.applied(safeStore);
+safeStore.notifyListeners(safeCommand, command);
+}
+}
+
+
+
@VisibleForTesting
public void load(Deps loading)
{
diff --git a/accord-core/src/main/java/accord/local/Cleanup.java
b/accord-core/src/main/java/accord/local/Cleanup.java
index 63ef7664..78dc341c 100644
--- a/accord-core/src/main/java/accord/local/Cleanup.java
+++ b/accord-core/src/main/java/accord/local/Cleanup.java
@@ -95,6 +95,7 @@ public enum Cleanup
return shouldCleanupInternal(txnId, status, durability, participants,
redundantBefore, durableBefore).filter(status);
}
+// TODO (required): simulate compaction of log records in burn test
@VisibleForImplementation
public static Cleanup shouldCleanupPartial(TxnId txnId, SaveStatus status,
Durability durability, StoreParticipants participants, RedundantBefore
redundantBefore, DurableBefore durableBefore)
{
diff --git a/accord-core/src/main/java/accord/local/Commands.java
b/accord-core/src/main/java/accord/local/Commands.java
index 637d791f..a9a2805c 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -484,7 +484,7 @@ public class Commands
* for transactions below a SyncPoint where we adopte
(cassandra-accord) 02/02: Override equals for snapshot
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 8690de597c4cddbd56e782f0f9ddcc46c15e406f
Author: Alex Petrov
AuthorDate: Fri Sep 27 10:49:07 2024 +0200
Override equals for snapshot
---
.../src/main/java/accord/local/CommandStores.java| 16
1 file changed, 16 insertions(+)
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java
b/accord-core/src/main/java/accord/local/CommandStores.java
index 5f70e794..545a7843 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -151,6 +152,21 @@ public abstract class CommandStores
this.epochs = epochs;
this.ranges = ranges;
}
+
+@Override
+public boolean equals(Object o)
+{
+if (this == o) return true;
+if (o == null || getClass() != o.getClass()) return false;
+Snapshot snapshot = (Snapshot) o;
+return Objects.deepEquals(epochs, snapshot.epochs) &&
Objects.deepEquals(ranges, snapshot.ranges);
+}
+
+@Override
+public int hashCode()
+{
+return Objects.hash(Arrays.hashCode(epochs),
Arrays.hashCode(ranges));
+}
}
final long[] epochs;
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 01/02: Overwrite reject before on every op
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit c8603983ec450fd501be919106e12ca181cf7dac
Author: Alex Petrov
AuthorDate: Fri Sep 27 08:55:03 2024 +0200
Overwrite reject before on every op
---
accord-core/src/main/java/accord/local/CommandStore.java | 11 +++
accord-core/src/main/java/accord/local/SafeCommandStore.java | 5 +++--
2 files changed, 6 insertions(+), 10 deletions(-)
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java
b/accord-core/src/main/java/accord/local/CommandStore.java
index f107eac3..3f450c26 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -281,13 +281,6 @@ public abstract class CommandStore implements AgentExecutor
redundantBefore = newRedundantBefore;
}
-protected void upsertRejectBefore(TxnId txnId, Ranges ranges)
-{
-ReducingRangeMap newRejectBefore = rejectBefore != null ?
rejectBefore : new ReducingRangeMap<>();
-newRejectBefore = ReducingRangeMap.add(newRejectBefore, ranges, txnId,
Timestamp::max);
-unsafeSetRejectBefore(newRejectBefore);
-}
-
/**
* This method may be invoked on a non-CommandStore thread
*/
@@ -330,7 +323,9 @@ public abstract class CommandStore implements AgentExecutor
{
// TODO (desired): narrow ranges to those that are owned
Invariants.checkArgument(txnId.kind() == ExclusiveSyncPoint);
-safeStore.upsertRejectBefore(txnId, ranges);
+ReducingRangeMap newRejectBefore = rejectBefore != null ?
rejectBefore : new ReducingRangeMap<>();
+newRejectBefore = ReducingRangeMap.add(newRejectBefore, ranges, txnId,
Timestamp::max);
+safeStore.setRejectBefore(newRejectBefore);
}
public final void markExclusiveSyncPointLocallyApplied(SafeCommandStore
safeStore, TxnId txnId, Ranges ranges)
diff --git a/accord-core/src/main/java/accord/local/SafeCommandStore.java
b/accord-core/src/main/java/accord/local/SafeCommandStore.java
index 58bf15f2..3178aaa9 100644
--- a/accord-core/src/main/java/accord/local/SafeCommandStore.java
+++ b/accord-core/src/main/java/accord/local/SafeCommandStore.java
@@ -44,6 +44,7 @@ import accord.primitives.Txn.Kind.Kinds;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
import static accord.local.Cleanup.NO;
import static accord.local.KeyHistory.COMMANDS;
@@ -274,9 +275,9 @@ public abstract class SafeCommandStore
commandStore().unsafeSetRangesForEpoch(rangesForEpoch);
}
-public void upsertRejectBefore(TxnId txnId, Ranges ranges)
+public void setRejectBefore(ReducingRangeMap next)
{
-commandStore().upsertRejectBefore(txnId, ranges);
+commandStore().unsafeSetRejectBefore(next);
}
public void updateCommandsForKey(Command prev, Command next)
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch CASSANDRA-19944-persistent-fields updated (b15aad72 -> 8690de59)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch CASSANDRA-19944-persistent-fields in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git from b15aad72 Fix build new c8603983 Overwrite reject before on every op new 8690de59 Override equals for snapshot The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: accord-core/src/main/java/accord/local/CommandStore.java | 11 +++ .../src/main/java/accord/local/CommandStores.java| 16 .../src/main/java/accord/local/SafeCommandStore.java | 5 +++-- 3 files changed, 22 insertions(+), 10 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch CASSANDRA-19944-persistent-fields updated: Fix build
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git The following commit(s) were added to refs/heads/CASSANDRA-19944-persistent-fields by this push: new b15aad72 Fix build b15aad72 is described below commit b15aad726d44e57f2ececcb996782fc518a6b4a8 Author: Alex Petrov AuthorDate: Fri Sep 27 08:26:06 2024 +0200 Fix build --- accord-core/src/main/java/accord/impl/InMemoryCommandStore.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java index 1109f2a6..0a900ee5 100644 --- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java +++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java @@ -56,12 +56,10 @@ import accord.local.Command; import accord.local.CommandStore; import accord.local.CommandStores.RangesForEpoch; import accord.local.Commands; -import accord.local.DurableBefore; import accord.local.KeyHistory; import accord.local.Node; import accord.local.NodeTimeService; import accord.local.PreLoadContext; -import accord.local.RedundantBefore; import accord.local.RedundantStatus; import accord.local.SafeCommand; import accord.local.SafeCommandStore; - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch CASSANDRA-19944-persistent-fields updated: Remove assert
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
The following commit(s) were added to
refs/heads/CASSANDRA-19944-persistent-fields by this push:
new 20b6a801 Remove assert
20b6a801 is described below
commit 20b6a801dc940a235a55c8b87c2763f1c36ea6e0
Author: Alex Petrov
AuthorDate: Thu Sep 26 22:22:08 2024 +0200
Remove assert
---
accord-core/src/main/java/accord/local/CommandStores.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java
b/accord-core/src/main/java/accord/local/CommandStores.java
index 34178574..5f70e794 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -709,7 +709,6 @@ public abstract class CommandStores
{
ShardHolder[] shards = current.shards;
CommandStore[] all = new CommandStore[shards.length];
-if (shards.length == 0) throw illegalState("Unable to get
CommandStore; non defined");
for (int i = 0; i < shards.length; i++)
all[i] = shards[i].store;
return all;
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 08/10: Make it easier to log changes in rangesForEpoch, durableBefore, redundantBefore, safeToRead, and rangesForEpoch
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 1f6d998835c64e7a4f6ac8bc23c5e052760ac493
Author: Alex Petrov
AuthorDate: Thu Sep 26 15:59:36 2024 +0200
Make it easier to log changes in rangesForEpoch, durableBefore,
redundantBefore, safeToRead, and rangesForEpoch
---
.../java/accord/impl/InMemoryCommandStore.java | 13 ++-
.../src/main/java/accord/local/CommandStore.java | 94 --
.../main/java/accord/local/SafeCommandStore.java | 31 +++
.../java/accord/messages/SetGloballyDurable.java | 2 +-
4 files changed, 94 insertions(+), 46 deletions(-)
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index 6ba93447..e66b25a9 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -479,8 +479,8 @@ public abstract class InMemoryCommandStore extends
CommandStore
{
if (current != null)
throw illegalState("Another operation is in progress or it's store
was not cleared");
-RangesForEpoch rangesForEpoch = updateRangesForEpoch();
current = createSafeStore(context, rangesForEpoch);
+updateRangesForEpoch(current);
return current;
}
@@ -663,7 +663,7 @@ public abstract class InMemoryCommandStore extends
CommandStore
protected final Map commands;
private final Map
timestampsForKey;
private final Map
commandsForKey;
-private final RangesForEpoch ranges;
+private RangesForEpoch ranges;
public InMemorySafeStore(InMemoryCommandStore commandStore,
RangesForEpoch ranges,
@@ -677,7 +677,7 @@ public abstract class InMemoryCommandStore extends
CommandStore
this.commands = commands;
this.commandsForKey = commandsForKey;
this.timestampsForKey = timestampsForKey;
-this.ranges = Invariants.nonNull(ranges);
+this.ranges = ranges;
}
@Override
@@ -797,6 +797,13 @@ public abstract class InMemoryCommandStore extends
CommandStore
return ranges;
}
+@Override
+public void setRangesForEpoch(RangesForEpoch rangesForEpoch)
+{
+super.setRangesForEpoch(rangesForEpoch);
+ranges = rangesForEpoch;
+}
+
@Override
public NodeTimeService time()
{
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java
b/accord-core/src/main/java/accord/local/CommandStore.java
index 966c4418..2ebe1c44 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -33,7 +33,6 @@ import java.util.function.Function;
import java.util.function.Supplier;
import javax.annotation.Nullable;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSortedMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -209,20 +208,19 @@ public abstract class CommandStore implements
AgentExecutor
return agent;
}
-public RangesForEpoch updateRangesForEpoch()
+public void updateRangesForEpoch(SafeCommandStore safeStore)
{
EpochUpdate update = epochUpdateHolder.get();
if (update == null)
-return rangesForEpoch;
+return;
update = epochUpdateHolder.getAndSet(null);
if (!update.addGlobalRanges.isEmpty())
-upsertDurableBefore(DurableBefore.create(update.addGlobalRanges,
TxnId.NONE, TxnId.NONE));
+
safeStore.upsertDurableBefore(DurableBefore.create(update.addGlobalRanges,
TxnId.NONE, TxnId.NONE));
if (update.addRedundantBefore.size() > 0)
-upsertRedundantBefore(update.addRedundantBefore);
+safeStore.upsertRedundantBefore(update.addRedundantBefore);
if (update.newRangesForEpoch != null)
-rangesForEpoch = update.newRangesForEpoch;
-return rangesForEpoch;
+safeStore.setRangesForEpoch(update.newRangesForEpoch);
}
public RangesForEpoch unsafeRangesForEpoch()
@@ -250,8 +248,7 @@ public abstract class CommandStore implements AgentExecutor
protected abstract void registerHistoricalTransactions(Deps deps,
SafeCommandStore safeStore);
-// implementations are expected to override this for persistence
-public void upsertDurableBefore(DurableBefore addDurableBefore)
+protected void upsertDurableBefore(DurableBefore addDurableBefore)
{
durableBefore = DurableBefore.merge(durableBefore, addDurableBefore);
}
@@ -261,6 +258,7 @@ public abstract class CommandStore implem
(cassandra-accord) 09/10: Simplify bootstrap
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit cff185a92a97722a1b42866b3cf245f899b46b9f
Author: Alex Petrov
AuthorDate: Thu Sep 26 20:16:35 2024 +0200
Simplify bootstrap
---
.../java/accord/impl/InMemoryCommandStore.java | 36 --
.../src/main/java/accord/local/CommandStore.java | 11 +--
.../main/java/accord/local/SafeCommandStore.java | 4 +--
3 files changed, 10 insertions(+), 41 deletions(-)
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index e66b25a9..1109f2a6 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -1468,40 +1468,4 @@ public abstract class InMemoryCommandStore extends
CommandStore
historicalRangeCommands.merge(txnId, ranges.slice(allRanges),
Ranges::with);
});
}
-
-@Override
-public void unsafeSetRangesForEpoch(RangesForEpoch newRangesForEpoch)
-{
-super.unsafeSetRangesForEpoch(newRangesForEpoch);
-}
-
-@Override
-public void unsafeSetDurableBefore(DurableBefore newDurableBefore)
-{
-super.unsafeSetDurableBefore(newDurableBefore);
-}
-
-@Override
-public void unsafeSetRedundantBefore(RedundantBefore newRedundantBefore)
-{
-super.unsafeSetRedundantBefore(newRedundantBefore);
-}
-
-@Override
-public void unsafeSetRejectBefore(ReducingRangeMap
newRejectBefore)
-{
-super.unsafeSetRejectBefore(newRejectBefore);
-}
-
-@Override
-public void unsafeSetSafeToRead(NavigableMap
newSafeToRead)
-{
-super.unsafeSetSafeToRead(newSafeToRead);
-}
-
-@Override
-public void unsafeSetBootstrapBeganAt(NavigableMap
newBootstrapBeganAt)
-{
-super.unsafeSetBootstrapBeganAt(newBootstrapBeganAt);
-}
}
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java
b/accord-core/src/main/java/accord/local/CommandStore.java
index 2ebe1c44..65d7cd5a 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -289,8 +289,14 @@ public abstract class CommandStore implements AgentExecutor
this.safeToRead = newSafeToRead;
}
+public void upsertSetBootstrapBeganAt(TxnId globalSyncId, Ranges ranges)
+{
+unsafeSetBootstrapBeganAt(bootstrap(globalSyncId, ranges,
bootstrapBeganAt));
+}
+
protected void unsafeSetBootstrapBeganAt(NavigableMap
newBootstrapBeganAt)
{
+
this.bootstrapBeganAt = newBootstrapBeganAt;
}
@@ -518,7 +524,7 @@ public abstract class CommandStore implements AgentExecutor
final void markBootstrapping(SafeCommandStore safeStore, TxnId
globalSyncId, Ranges ranges)
{
-safeStore.upsertSetBootstrapBeganAt(bootstrap(globalSyncId, ranges,
bootstrapBeganAt));
+safeStore.upsertSetBootstrapBeganAt(globalSyncId, ranges);
RedundantBefore addRedundantBefore = RedundantBefore.create(ranges,
Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, TxnId.NONE,
globalSyncId);
safeStore.upsertRedundantBefore(addRedundantBefore);
safeStore.upsertDurableBefore(DurableBefore.create(ranges, TxnId.NONE,
TxnId.NONE));
@@ -576,7 +582,6 @@ public abstract class CommandStore implements AgentExecutor
safeStore.upsertRedundantBefore(addRedundantBefore);
// find which ranges need to bootstrap, subtracting those already in
progress that cover the id
-// safeStore.upsertUnsafeToRead(ranges);
markUnsafeToRead(ranges);
}
@@ -795,7 +800,7 @@ public abstract class CommandStore implements AgentExecutor
});
}
-protected static ImmutableSortedMap bootstrap(TxnId at,
Ranges ranges, NavigableMap bootstrappedAt)
+public static ImmutableSortedMap bootstrap(TxnId at, Ranges
ranges, NavigableMap bootstrappedAt)
{
Invariants.checkArgument(bootstrappedAt.lastKey().compareTo(at) < 0 ||
at == TxnId.NONE);
if (at == TxnId.NONE)
diff --git a/accord-core/src/main/java/accord/local/SafeCommandStore.java
b/accord-core/src/main/java/accord/local/SafeCommandStore.java
index 82ba1c06..58bf15f2 100644
--- a/accord-core/src/main/java/accord/local/SafeCommandStore.java
+++ b/accord-core/src/main/java/accord/local/SafeCommandStore.java
@@ -254,9 +254,9 @@ public abstract class SafeCommandStore
commandStore().upsertRedundantBefore(addRedundantBefore);
}
-public void upsertSetBootstrapBeganAt(NavigableMap
newBootstrapBeganAt)
+public void upsertSetBootstrapBeganAt(TxnId globalSyncId, Ranges ranges)
{
-commandStore().unsafeSetBootstrapBeganAt(newBootstrapBega
(cassandra-accord) 05/10: wip: journal replay finishing touches
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit c6b2cbaac852c5c2453ff9b289db0866ccfa0345
Author: Benedict Elliott Smith
AuthorDate: Wed Sep 25 18:23:10 2024 +0100
wip: journal replay finishing touches
---
.../src/main/java/accord/api/DataStore.java| 3 +-
.../java/accord/impl/InMemoryCommandStore.java | 48 +---
.../src/main/java/accord/local/CommandStore.java | 85 ++
.../accord/impl/list/ListFetchCoordinator.java | 5 +-
.../src/test/java/accord/impl/list/ListStore.java | 12 ++-
.../src/test/java/accord/impl/mock/MockStore.java | 8 ++
.../main/java/accord/maelstrom/MaelstromStore.java | 9 +++
7 files changed, 109 insertions(+), 61 deletions(-)
diff --git a/accord-core/src/main/java/accord/api/DataStore.java
b/accord-core/src/main/java/accord/api/DataStore.java
index a93ff073..45d62101 100644
--- a/accord-core/src/main/java/accord/api/DataStore.java
+++ b/accord-core/src/main/java/accord/api/DataStore.java
@@ -25,6 +25,7 @@ import accord.local.SafeCommandStore;
import accord.primitives.Ranges;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
@@ -112,6 +113,6 @@ public interface DataStore
}
FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges,
SyncPoint syncPoint, FetchRanges callback);
-default AsyncResult snapshot(Ranges ranges) { return
AsyncResults.success(null); };
+AsyncResult snapshot(Ranges ranges, TxnId before);
default void restoreFromSnapshot() {};
}
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index 1b474779..2793b6e9 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -56,10 +56,12 @@ import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores.RangesForEpoch;
import accord.local.Commands;
+import accord.local.DurableBefore;
import accord.local.KeyHistory;
import accord.local.Node;
import accord.local.NodeTimeService;
import accord.local.PreLoadContext;
+import accord.local.RedundantBefore;
import accord.local.RedundantStatus;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
@@ -84,6 +86,7 @@ import accord.primitives.Timestamp;
import accord.primitives.Txn.Kind.Kinds;
import accord.primitives.TxnId;
import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
@@ -428,15 +431,6 @@ public abstract class InMemoryCommandStore extends
CommandStore
return new InMemorySafeStore(this, ranges, context, commands,
timestampsForKey, commandsForKeys);
}
-private void loadCommandsForKey(RoutableKey key,
-KeyHistory keyHistory,
-Map timestampsForKey,
-Map commandsForKey)
-{
-commandsForKey.put(key, commandsForKey((Key)
key).createSafeReference());
-timestampsForKey.put(key, timestampsForKey((Key)
key).createSafeReference());
-}
-
protected void validateRead(Command current) {}
protected final InMemorySafeStore createSafeStore(PreLoadContext context,
RangesForEpoch ranges)
@@ -1466,4 +1460,40 @@ public abstract class InMemoryCommandStore extends
CommandStore
historicalRangeCommands.merge(txnId, ranges.slice(allRanges),
Ranges::with);
});
}
+
+@Override
+public void unsafeSetRangesForEpoch(RangesForEpoch newRangesForEpoch)
+{
+super.unsafeSetRangesForEpoch(newRangesForEpoch);
+}
+
+@Override
+public void unsafeSetDurableBefore(DurableBefore newDurableBefore)
+{
+super.unsafeSetDurableBefore(newDurableBefore);
+}
+
+@Override
+public void unsafeSetRedundantBefore(RedundantBefore newRedundantBefore)
+{
+super.unsafeSetRedundantBefore(newRedundantBefore);
+}
+
+@Override
+public void unsafeSetRejectBefore(ReducingRangeMap
newRejectBefore)
+{
+super.unsafeSetRejectBefore(newRejectBefore);
+}
+
+@Override
+public void unsafeSetSafeToRead(NavigableMap
newSafeToRead)
+{
+super.unsafeSetSafeToRead(newSafeToRead);
+}
+
+@Override
+public void unsafeSetBootstrapBeganAt(NavigableMap
newBootstrapBeganAt)
+{
+super.unsafeSetBootstrapBeganAt(newBootstrapBeganAt);
+}
}
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java
b/accord-core/src/main/java/accord/local/CommandStore.java
index 298d0c04
(cassandra-accord) 10/10: Make it compile
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 11c49113f1bb599a90e1d9be48e351f4c2a06051
Author: Alex Petrov
AuthorDate: Thu Sep 26 20:47:36 2024 +0200
Make it compile
---
.../src/main/java/accord/local/CommandStore.java| 13 +
.../src/main/java/accord/local/CommandStores.java | 17 +
2 files changed, 30 insertions(+)
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java
b/accord-core/src/main/java/accord/local/CommandStore.java
index 65d7cd5a..f107eac3 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -33,6 +33,7 @@ import java.util.function.Function;
import java.util.function.Supplier;
import javax.annotation.Nullable;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSortedMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -208,6 +209,12 @@ public abstract class CommandStore implements AgentExecutor
return agent;
}
+@VisibleForTesting // TODO: check if this is right
+public void updateRangesForEpoch()
+{
+execute(empty(), this::updateRangesForEpoch).beginAsResult();
+}
+
public void updateRangesForEpoch(SafeCommandStore safeStore)
{
EpochUpdate update = epochUpdateHolder.get();
@@ -634,6 +641,12 @@ public abstract class CommandStore implements AgentExecutor
return durableBefore;
}
+@VisibleForTesting
+public final NavigableMap bootstrapBeganAt() { return
bootstrapBeganAt; }
+
+@VisibleForTesting
+public NavigableMap safeToRead() { return safeToRead; }
+
public final boolean isRejectedIfNotPreAccepted(TxnId txnId,
Unseekables participants)
{
if (rejectBefore == null)
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java
b/accord-core/src/main/java/accord/local/CommandStores.java
index a6b7b030..34178574 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -141,6 +141,18 @@ public abstract class CommandStores
// TODO (expected): merge with RedundantBefore, and standardise
executeRanges() to treat removing stale ranges the same as adding new epoch
ranges
public static class RangesForEpoch
{
+public static class Snapshot
+{
+public final long[] epochs;
+public final Ranges[] ranges;
+
+public Snapshot(long[] epochs, Ranges[] ranges)
+{
+this.epochs = epochs;
+this.ranges = ranges;
+}
+}
+
final long[] epochs;
final Ranges[] ranges;
final CommandStore store;
@@ -159,6 +171,11 @@ public abstract class CommandStores
this.store = store;
}
+public Snapshot snapshot()
+{
+return new Snapshot(epochs, ranges);
+}
+
public RangesForEpoch withRanges(long epoch, Ranges latestRanges)
{
Invariants.checkArgument(epochs.length == 0 ||
epochs[epochs.length - 1] <= epoch);
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 02/10: Fixes after rebase and an attempt to reconstruct redundant before
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit aa1d855c9e8863a0c4e72f255e26a5b161b6767d
Author: Alex Petrov
AuthorDate: Tue Sep 24 22:44:30 2024 +0200
Fixes after rebase and an attempt to reconstruct redundant before
---
.../java/accord/impl/InMemoryCommandStore.java | 58 +-
.../src/main/java/accord/local/Bootstrap.java | 32 ++--
.../src/main/java/accord/local/Cleanup.java| 2 +-
.../src/main/java/accord/local/CommandStore.java | 43 ++--
.../src/main/java/accord/local/Commands.java | 4 +-
.../main/java/accord/local/SafeCommandStore.java | 2 +-
.../src/test/java/accord/burn/BurnTest.java| 3 +-
.../accord/impl/basic/DelayedCommandStores.java| 1 -
8 files changed, 83 insertions(+), 62 deletions(-)
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index c74e9f27..65bc1618 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -42,6 +42,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSortedMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,16 +51,19 @@ import accord.api.DataStore;
import accord.api.Key;
import accord.api.LocalListeners;
import accord.api.ProgressLog;
+import accord.api.Scheduler;
import accord.impl.progresslog.DefaultProgressLog;
import accord.local.Cleanup;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores.RangesForEpoch;
import accord.local.Commands;
+import accord.local.DurableBefore;
import accord.local.KeyHistory;
import accord.local.Node;
import accord.local.NodeTimeService;
import accord.local.PreLoadContext;
+import accord.local.RedundantBefore;
import accord.local.RedundantStatus;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
@@ -127,16 +131,36 @@ public abstract class InMemoryCommandStore extends
CommandStore
// To simulate the delay in simulatedAsyncPersist
private final Scheduler scheduler;
-private static FieldPersister
simulatedAsyncPersistFactory(Scheduler scheduler)
-{
-return (commandStore, toPersist) -> simulatedAsyncPersist(scheduler,
commandStore, toPersist);
-}
-private static AsyncResult simulatedAsyncPersist(Scheduler
scheduler, CommandStore store, T toPersist)
+private static final class SimulatedFieldPersister implements
FieldPersister
{
-AsyncResult.Settable result = AsyncResults.settable();
-scheduler.once(() -> result.trySuccess(null), 100,
TimeUnit.MICROSECONDS);
-return result;
+private T lastValue;
+private final Scheduler scheduler;
+private final Node node;
+private final int id;
+public SimulatedFieldPersister(Scheduler scheduler, T defaultValue,
Node node, int id)
+{
+this.scheduler = scheduler;
+this.lastValue = defaultValue;
+this.node = node;
+this.id = id;
+}
+
+public AsyncResult persist(CommandStore store, T toPersist)
+{
+System.out.println("Persisting for " + node.id() + "-store-" + id);
+AsyncResult.Settable result = AsyncResults.settable();
+scheduler.once(() -> {
+lastValue = toPersist;
+result.trySuccess(null);
+}, 100, TimeUnit.MICROSECONDS);
+return result;
+}
+
+public T restore()
+{
+return lastValue;
+}
}
public InMemoryCommandStore(int id, NodeTimeService time, Agent agent,
DataStore store, ProgressLog.Factory progressLogFactory, LocalListeners.Factory
listenersFactory, EpochUpdateHolder epochUpdateHolder, Scheduler scheduler)
@@ -148,10 +172,10 @@ public abstract class InMemoryCommandStore extends
CommandStore
progressLogFactory,
listenersFactory,
epochUpdateHolder,
- simulatedAsyncPersistFactory(scheduler),
- simulatedAsyncPersistFactory(scheduler),
- simulatedAsyncPersistFactory(scheduler),
- simulatedAsyncPersistFactory(scheduler));
+ new SimulatedFieldPersister<>(scheduler, DurableBefore.EMPTY,
(Node) time, id),
+ new SimulatedFieldPersister<>(scheduler, RedundantBefore.EMPTY,
(Node) time, id),
+ new SimulatedFieldPersister<>(scheduler,
ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY), (Node) time, id),
+
(cassandra-accord) 04/10: temporary fix for invalid use of Sets.filter
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit e7744d0d6ce882dad855f969e19196289823beeb
Author: Benedict Elliott Smith
AuthorDate: Fri Sep 20 12:09:37 2024 +0100
temporary fix for invalid use of Sets.filter
---
.../coordinate/tracking/AbstractTracker.java | 2 +-
.../src/main/java/accord/topology/Topologies.java | 41 ++
.../src/main/java/accord/topology/Topology.java| 5 ---
3 files changed, 28 insertions(+), 20 deletions(-)
diff --git
a/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
b/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
index e58f3c67..183b4879 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
@@ -180,7 +180,7 @@ public abstract class AbstractTracker
return topologies.nodes();
}
-public Set nonStaleNodes()
+public Collection nonStaleNodes()
{
return topologies.nonStaleNodes();
}
diff --git a/accord-core/src/main/java/accord/topology/Topologies.java
b/accord-core/src/main/java/accord/topology/Topologies.java
index 95fce506..91a0d7ce 100644
--- a/accord-core/src/main/java/accord/topology/Topologies.java
+++ b/accord-core/src/main/java/accord/topology/Topologies.java
@@ -18,11 +18,15 @@
package accord.topology;
+import java.util.AbstractCollection;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import com.google.common.collect.Iterators;
import com.google.common.collect.Sets;
import accord.api.TopologySorter;
@@ -74,7 +78,29 @@ public interface Topologies extends TopologySorter
// note this can be expensive to evaluate
SortedList nodes();
-Set nonStaleNodes();
+default Collection nonStaleNodes()
+{
+Topology currentTopology = current();
+Set staleIds = currentTopology.staleIds();
+SortedList nodes = nodes();
+if (staleIds.isEmpty())
+return nodes;
+
+return new AbstractCollection<>()
+{
+@Override
+public Iterator iterator()
+{
+return Iterators.filter(nodes.iterator(), id ->
!staleIds.contains(id));
+}
+
+@Override
+public int size()
+{
+return Iterators.size(iterator());
+}
+};
+}
int estimateUniqueNodes();
@@ -216,12 +242,6 @@ public interface Topologies extends TopologySorter
return topology.nodes();
}
-@Override
-public Set nonStaleNodes()
-{
-return topology.nonStaleNodes();
-}
-
@Override
public int estimateUniqueNodes()
{
@@ -402,13 +422,6 @@ public interface Topologies extends TopologySorter
return result;
}
-@Override
-public Set nonStaleNodes()
-{
-Topology currentTopology = current();
-return Sets.filter(nodes(), id ->
!currentTopology.staleIds().contains(id));
-}
-
@Override
public Ranges computeRangesForNode(Id node)
{
diff --git a/accord-core/src/main/java/accord/topology/Topology.java
b/accord-core/src/main/java/accord/topology/Topology.java
index f1fc988f..9f639f83 100644
--- a/accord-core/src/main/java/accord/topology/Topology.java
+++ b/accord-core/src/main/java/accord/topology/Topology.java
@@ -558,11 +558,6 @@ public class Topology
return nodeIds;
}
-public Set nonStaleNodes()
-{
-return Sets.filter(nodes(), id -> !staleIds.contains(id));
-}
-
public Ranges ranges()
{
return subsetOfRanges;
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch CASSANDRA-19944-persistent-fields created (now 11c49113)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch CASSANDRA-19944-persistent-fields in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git at 11c49113 Make it compile This branch includes the following new commits: new cc6c12cb Persists metadata syncrhonously new aa1d855c Fixes after rebase and an attempt to reconstruct redundant before new cf62c307 wip new e7744d0d temporary fix for invalid use of Sets.filter new c6b2cbaa wip: journal replay finishing touches new 4fb7025d wip: journal replay finishing touches new e93d5d23 Checkstyle new 1f6d9988 Make it easier to log changes in rangesForEpoch, durableBefore, redundantBefore, safeToRead, and rangesForEpoch new cff185a9 Simplify bootstrap new 11c49113 Make it compile The 10 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 07/10: Checkstyle
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git commit e93d5d236f7b8f145d2c3aae1a396b64eaf9a7ba Author: Alex Petrov AuthorDate: Thu Sep 26 14:13:04 2024 +0200 Checkstyle --- accord-core/src/main/java/accord/api/DataStore.java | 1 - accord-core/src/main/java/accord/local/Commands.java | 2 -- accord-core/src/main/java/accord/topology/Topology.java | 5 ++--- accord-core/src/test/java/accord/impl/basic/Cluster.java | 2 -- .../src/test/java/accord/impl/basic/DelayedCommandStores.java| 1 - 5 files changed, 2 insertions(+), 9 deletions(-) diff --git a/accord-core/src/main/java/accord/api/DataStore.java b/accord-core/src/main/java/accord/api/DataStore.java index e0d013d8..623b021f 100644 --- a/accord-core/src/main/java/accord/api/DataStore.java +++ b/accord-core/src/main/java/accord/api/DataStore.java @@ -27,7 +27,6 @@ import accord.primitives.SyncPoint; import accord.primitives.Timestamp; import accord.primitives.TxnId; import accord.utils.async.AsyncResult; -import accord.utils.async.AsyncResults; /** * A marker interface for a shard instance's storage, that is passed to diff --git a/accord-core/src/main/java/accord/local/Commands.java b/accord-core/src/main/java/accord/local/Commands.java index e164ec11..3b4d7c96 100644 --- a/accord-core/src/main/java/accord/local/Commands.java +++ b/accord-core/src/main/java/accord/local/Commands.java @@ -75,9 +75,7 @@ import static accord.local.Status.PreApplied; import static accord.local.Status.PreCommitted; import static accord.local.Status.Stable; import static accord.local.Status.Truncated; -import static accord.primitives.Routables.Slice.Minimal; import static accord.primitives.Route.isFullRoute; -import static accord.primitives.Txn.Kind.ExclusiveSyncPoint; import static accord.utils.Invariants.illegalState; public class Commands diff --git a/accord-core/src/main/java/accord/topology/Topology.java b/accord-core/src/main/java/accord/topology/Topology.java index 9f639f83..fca2618b 100644 --- a/accord-core/src/main/java/accord/topology/Topology.java +++ b/accord-core/src/main/java/accord/topology/Topology.java @@ -34,9 +34,6 @@ import java.util.stream.IntStream; import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Sets; -import org.agrona.collections.Int2ObjectHashMap; -import org.agrona.collections.IntArrayList; import accord.api.RoutingKey; import accord.local.Node.Id; @@ -53,6 +50,8 @@ import accord.utils.IndexedTriFunction; import accord.utils.SimpleBitSet; import accord.utils.SortedArrays.SortedArrayList; import accord.utils.Utils; +import org.agrona.collections.Int2ObjectHashMap; +import org.agrona.collections.IntArrayList; import static accord.utils.Invariants.illegalArgument; import static accord.utils.SortedArrays.Search.FLOOR; diff --git a/accord-core/src/test/java/accord/impl/basic/Cluster.java b/accord-core/src/test/java/accord/impl/basic/Cluster.java index 8aa3817d..a938052e 100644 --- a/accord-core/src/test/java/accord/impl/basic/Cluster.java +++ b/accord-core/src/test/java/accord/impl/basic/Cluster.java @@ -67,7 +67,6 @@ import accord.impl.CoordinateDurabilityScheduling; import accord.impl.DefaultLocalListeners; import accord.impl.DefaultRemoteListeners; import accord.impl.DefaultRequestTimeouts; -import accord.impl.InMemoryCommandStore; import accord.impl.InMemoryCommandStore.GlobalCommand; import accord.impl.MessageListener; import accord.impl.PrefixedIntHashKey; @@ -104,7 +103,6 @@ import accord.topology.TopologyRandomizer; import accord.utils.Gens; import accord.utils.Invariants; import accord.utils.RandomSource; -import accord.utils.ReducingRangeMap; import accord.utils.async.AsyncChains; import accord.utils.async.AsyncResult; diff --git a/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java b/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java index fd2256a3..0de6debc 100644 --- a/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java +++ b/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java @@ -40,7 +40,6 @@ import accord.api.Agent; import accord.api.DataStore; import accord.api.LocalListeners; import accord.api.ProgressLog; -import accord.api.Scheduler; import accord.impl.InMemoryCommandStore; import accord.impl.InMemoryCommandStores; import accord.impl.InMemorySafeCommand; - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 06/10: wip: journal replay finishing touches
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 4fb7025dd32f1e743cf19311356a67e6613a3db4
Author: Benedict Elliott Smith
AuthorDate: Wed Sep 25 19:15:48 2024 +0100
wip: journal replay finishing touches
---
accord-core/src/main/java/accord/api/DataStore.java| 1 -
accord-core/src/main/java/accord/impl/InMemoryCommandStore.java| 1 +
accord-core/src/main/java/accord/topology/Topologies.java | 1 -
accord-core/src/test/java/accord/impl/basic/Cluster.java | 2 ++
.../src/test/java/accord/impl/list/ListFetchCoordinator.java | 2 +-
accord-core/src/test/java/accord/impl/list/ListStore.java | 7 ---
6 files changed, 4 insertions(+), 10 deletions(-)
diff --git a/accord-core/src/main/java/accord/api/DataStore.java
b/accord-core/src/main/java/accord/api/DataStore.java
index 45d62101..e0d013d8 100644
--- a/accord-core/src/main/java/accord/api/DataStore.java
+++ b/accord-core/src/main/java/accord/api/DataStore.java
@@ -114,5 +114,4 @@ public interface DataStore
FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges,
SyncPoint syncPoint, FetchRanges callback);
AsyncResult snapshot(Ranges ranges, TxnId before);
-default void restoreFromSnapshot() {};
}
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index 2793b6e9..6ba93447 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -1323,6 +1323,7 @@ public abstract class InMemoryCommandStore extends
CommandStore
commandsForKey.clear();
rangeCommands.clear();
historicalRangeCommands.clear();
+unsafeSetRejectBefore(new ReducingRangeMap<>());
}
public interface Loader
diff --git a/accord-core/src/main/java/accord/topology/Topologies.java
b/accord-core/src/main/java/accord/topology/Topologies.java
index 91a0d7ce..2d4a3494 100644
--- a/accord-core/src/main/java/accord/topology/Topologies.java
+++ b/accord-core/src/main/java/accord/topology/Topologies.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Set;
import com.google.common.collect.Iterators;
-import com.google.common.collect.Sets;
import accord.api.TopologySorter;
import accord.local.Node;
diff --git a/accord-core/src/test/java/accord/impl/basic/Cluster.java
b/accord-core/src/test/java/accord/impl/basic/Cluster.java
index a938052e..8aa3817d 100644
--- a/accord-core/src/test/java/accord/impl/basic/Cluster.java
+++ b/accord-core/src/test/java/accord/impl/basic/Cluster.java
@@ -67,6 +67,7 @@ import accord.impl.CoordinateDurabilityScheduling;
import accord.impl.DefaultLocalListeners;
import accord.impl.DefaultRemoteListeners;
import accord.impl.DefaultRequestTimeouts;
+import accord.impl.InMemoryCommandStore;
import accord.impl.InMemoryCommandStore.GlobalCommand;
import accord.impl.MessageListener;
import accord.impl.PrefixedIntHashKey;
@@ -103,6 +104,7 @@ import accord.topology.TopologyRandomizer;
import accord.utils.Gens;
import accord.utils.Invariants;
import accord.utils.RandomSource;
+import accord.utils.ReducingRangeMap;
import accord.utils.async.AsyncChains;
import accord.utils.async.AsyncResult;
diff --git
a/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
b/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
index a4687514..c7ffc2c6 100644
--- a/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
+++ b/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
@@ -65,7 +65,7 @@ public class ListFetchCoordinator extends
AbstractFetchCoordinator
ListData listData = (ListData) data;
persisting.add(commandStore.execute(PreLoadContext.empty(), safeStore
-> {
listData.forEach((key, value) -> listStore.data.merge(key, value,
Timestamped::merge));
-}).addCallback((ignore, fail) -> {
+}).flatMap(ignore -> listStore.snapshot(received,
syncPoint.syncId)).addCallback((success, fail) -> {
if (fail == null) success(from, received);
else fail(from, received, fail);
}).beginAsResult());
diff --git a/accord-core/src/test/java/accord/impl/list/ListStore.java
b/accord-core/src/test/java/accord/impl/list/ListStore.java
index 2473d6dc..71aecfde 100644
--- a/accord-core/src/test/java/accord/impl/list/ListStore.java
+++ b/accord-core/src/test/java/accord/impl/list/ListStore.java
@@ -41,13 +41,8 @@ import accord.coordinate.Timeout;
import accord.coordinate.TopologyMismatch;
import accord.coordinate.tracking.AllTracker;
import accord.coordinate.tracking.RequestStatus;
-import accord.impl.InMemoryCommandStore;
import accord.impl
(cassandra-accord) 03/10: wip
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit cf62c3071c7a0fbdb50598f4b98e8f296b3dbc46
Author: Benedict Elliott Smith
AuthorDate: Wed Sep 25 17:20:12 2024 +0100
wip
---
.../src/main/java/accord/api/DataStore.java| 3 +-
.../java/accord/impl/InMemoryCommandStore.java | 179 +-
.../java/accord/impl/InMemoryCommandStores.java| 17 +-
.../java/accord/impl/progresslog/HomeState.java| 2 +-
.../java/accord/impl/progresslog/WaitingState.java | 4 +-
.../src/main/java/accord/local/Bootstrap.java | 34 +--
.../src/main/java/accord/local/Cleanup.java| 3 +-
.../src/main/java/accord/local/Command.java| 3 +-
.../src/main/java/accord/local/CommandStore.java | 265 +
.../src/main/java/accord/local/CommandStores.java | 14 +-
.../src/main/java/accord/local/Commands.java | 51 ++--
accord-core/src/main/java/accord/local/Node.java | 2 +-
.../main/java/accord/local/RedundantBefore.java| 41 ++--
.../main/java/accord/local/SafeCommandStore.java | 2 +-
.../main/java/accord/local/cfk/CommandsForKey.java | 4 +-
.../src/main/java/accord/local/cfk/Updating.java | 6 +-
.../java/accord/messages/SetGloballyDurable.java | 2 +-
.../main/java/accord/messages/SetShardDurable.java | 2 +-
.../test/java/accord/impl/RemoteListenersTest.java | 7 +-
.../src/test/java/accord/impl/basic/Cluster.java | 2 +-
.../accord/impl/basic/DelayedCommandStores.java| 14 +-
.../src/test/java/accord/impl/list/ListStore.java | 107 +++--
.../java/accord/local/BootstrapLocalTxnTest.java | 131 --
.../java/accord/local/cfk/CommandsForKeyTest.java | 6 +-
24 files changed, 288 insertions(+), 613 deletions(-)
diff --git a/accord-core/src/main/java/accord/api/DataStore.java
b/accord-core/src/main/java/accord/api/DataStore.java
index 259b39cd..a93ff073 100644
--- a/accord-core/src/main/java/accord/api/DataStore.java
+++ b/accord-core/src/main/java/accord/api/DataStore.java
@@ -26,6 +26,7 @@ import accord.primitives.Ranges;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults;
/**
* A marker interface for a shard instance's storage, that is passed to
@@ -111,6 +112,6 @@ public interface DataStore
}
FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges,
SyncPoint syncPoint, FetchRanges callback);
-default void snapshot() {};
+default AsyncResult snapshot(Ranges ranges) { return
AsyncResults.success(null); };
default void restoreFromSnapshot() {};
}
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index 65bc1618..1b474779 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -42,7 +42,6 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableSortedMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,19 +50,16 @@ import accord.api.DataStore;
import accord.api.Key;
import accord.api.LocalListeners;
import accord.api.ProgressLog;
-import accord.api.Scheduler;
import accord.impl.progresslog.DefaultProgressLog;
import accord.local.Cleanup;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores.RangesForEpoch;
import accord.local.Commands;
-import accord.local.DurableBefore;
import accord.local.KeyHistory;
import accord.local.Node;
import accord.local.NodeTimeService;
import accord.local.PreLoadContext;
-import accord.local.RedundantBefore;
import accord.local.RedundantStatus;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
@@ -90,8 +86,6 @@ import accord.primitives.TxnId;
import accord.utils.Invariants;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
-import accord.utils.async.AsyncResult;
-import accord.utils.async.AsyncResults;
import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
import static accord.local.SafeCommandStore.TestDep.WITH;
@@ -129,54 +123,9 @@ public abstract class InMemoryCommandStore extends
CommandStore
private InMemorySafeStore current;
-// To simulate the delay in simulatedAsyncPersist
-private final Scheduler scheduler;
-
-private static final class SimulatedFieldPersister implements
FieldPersister
-{
-private T lastValue;
-private final Scheduler scheduler;
-private final Node node;
-private final int id;
-public SimulatedFieldPersister(Scheduler sche
(cassandra-accord) 01/10: Persists metadata syncrhonously
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19944-persistent-fields
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit cc6c12cb914a03826bfc65635cdd17317eade84e
Author: Ariel Weisberg
AuthorDate: Thu Sep 19 16:09:36 2024 -0400
Persists metadata syncrhonously
---
.../java/accord/impl/InMemoryCommandStore.java | 148 ++
.../java/accord/impl/InMemoryCommandStores.java| 24 +-
.../src/main/java/accord/local/Bootstrap.java | 12 +-
.../src/main/java/accord/local/CommandStore.java | 307 +
.../src/main/java/accord/local/CommandStores.java | 22 +-
.../src/main/java/accord/local/Commands.java | 40 ++-
accord-core/src/main/java/accord/local/Node.java | 2 +-
.../java/accord/messages/SetGloballyDurable.java | 5 +-
.../main/java/accord/messages/SetShardDurable.java | 2 +-
.../src/main/java/accord/utils/TriConsumer.java| 24 ++
.../main/java/accord/utils/async/AsyncChain.java | 11 +
.../test/java/accord/impl/RemoteListenersTest.java | 15 +-
.../accord/impl/basic/DelayedCommandStores.java| 16 +-
.../java/accord/local/BootstrapLocalTxnTest.java | 20 +-
.../java/accord/local/cfk/CommandsForKeyTest.java | 14 +-
15 files changed, 494 insertions(+), 168 deletions(-)
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index 872a6f25..c74e9f27 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -86,6 +86,8 @@ import accord.primitives.TxnId;
import accord.utils.Invariants;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults;
import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
import static accord.local.SafeCommandStore.TestDep.WITH;
@@ -123,9 +125,34 @@ public abstract class InMemoryCommandStore extends
CommandStore
private InMemorySafeStore current;
-public InMemoryCommandStore(int id, NodeTimeService time, Agent agent,
DataStore store, ProgressLog.Factory progressLogFactory, LocalListeners.Factory
listenersFactory, EpochUpdateHolder epochUpdateHolder)
+// To simulate the delay in simulatedAsyncPersist
+private final Scheduler scheduler;
+private static FieldPersister
simulatedAsyncPersistFactory(Scheduler scheduler)
{
-super(id, time, agent, store, progressLogFactory, listenersFactory,
epochUpdateHolder);
+return (commandStore, toPersist) -> simulatedAsyncPersist(scheduler,
commandStore, toPersist);
+}
+
+private static AsyncResult simulatedAsyncPersist(Scheduler
scheduler, CommandStore store, T toPersist)
+{
+AsyncResult.Settable result = AsyncResults.settable();
+scheduler.once(() -> result.trySuccess(null), 100,
TimeUnit.MICROSECONDS);
+return result;
+}
+
+public InMemoryCommandStore(int id, NodeTimeService time, Agent agent,
DataStore store, ProgressLog.Factory progressLogFactory, LocalListeners.Factory
listenersFactory, EpochUpdateHolder epochUpdateHolder, Scheduler scheduler)
+{
+super(id,
+ time,
+ agent,
+ store,
+ progressLogFactory,
+ listenersFactory,
+ epochUpdateHolder,
+ simulatedAsyncPersistFactory(scheduler),
+ simulatedAsyncPersistFactory(scheduler),
+ simulatedAsyncPersistFactory(scheduler),
+ simulatedAsyncPersistFactory(scheduler));
+this.scheduler = scheduler;
}
protected boolean canExposeUnloaded()
@@ -367,57 +394,59 @@ public abstract class InMemoryCommandStore extends
CommandStore
}
@Override
-public void markShardDurable(SafeCommandStore safeStore, TxnId syncId,
Ranges ranges)
+public AsyncChain markShardDurable(SafeCommandStore safeStore, TxnId
syncId, Ranges ranges)
{
-super.markShardDurable(safeStore, syncId, ranges);
-markShardDurable(syncId, ranges);
-}
-
-private void markShardDurable(TxnId syncId, Ranges ranges)
-{
-if (!rangeCommands.containsKey(syncId))
-historicalRangeCommands.merge(syncId, ranges, Ranges::with);
-
-// TODO (now): apply on retrieval
-historicalRangeCommands.entrySet().removeIf(next ->
next.getKey().compareTo(syncId) < 0 && next.getValue().intersects(ranges));
-rangeCommands.entrySet().removeIf(tx -> {
-if (tx.getKey().compareTo(syncId) >= 0)
-return false;
-Ranges newRanges = tx.getValue().ranges.without(ranges);
-if (!newRanges.isEmpty())
-{
-tx.getValue().ranges = newRanges;
-
(cassandra-accord) branch CASSANDRA-19877 updated (bf34e0b0 -> 626785a8)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch CASSANDRA-19877 in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git omit bf34e0b0 Compaction add 486cd4bc Ninja fix: RandomSource.asJdkRandom did not provide a seed. This should have been fixed in CASSANDRA-19847 as it was fixed on Cassadnra trunk add 626785a8 Compaction This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (bf34e0b0) \ N -- N -- N refs/heads/CASSANDRA-19877 (626785a8) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: accord-core/src/main/java/accord/utils/RandomSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch CASSANDRA-19866 deleted (was dd51039d0b)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch CASSANDRA-19866 in repository https://gitbox.apache.org/repos/asf/cassandra.git was dd51039d0b Fix Journal segment allocation/switch race condition The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository. - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 01/01: Compaction
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19877
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit bf34e0b079700a3866bbd23cdb2c31ee6fa02268
Author: Alex Petrov
AuthorDate: Mon Sep 16 21:27:46 2024 +0200
Compaction
---
accord-core/src/main/java/accord/local/Commands.java | 16
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/accord-core/src/main/java/accord/local/Commands.java
b/accord-core/src/main/java/accord/local/Commands.java
index 43b8d77f..11d3fa08 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -851,6 +851,17 @@ public class Commands
|| (command.route() == null ||
Infer.safeToCleanup(safeStore, command, command.route(), command.executeAt())
|| safeStore.isFullyPreBootstrapOrStale(command,
command.route().participants()))
, "Command %s could not be truncated", command);
+Command result = purge(command, maybeFullRoute, cleanup);
+
+safeCommand.update(safeStore, result);
+safeStore.progressLog().clear(safeCommand.txnId());
+if (notifyListeners)
+safeStore.notifyListeners(safeCommand, command);
+
+return result;
+}
+public static Command purge(Command command, @Nullable Unseekables
maybeFullRoute, Cleanup cleanup)
+{
Command result;
switch (cleanup)
{
@@ -878,10 +889,7 @@ public class Commands
break;
}
-safeCommand.update(safeStore, result);
-safeStore.progressLog().clear(safeCommand.txnId());
-if (notifyListeners)
-safeStore.notifyListeners(safeCommand, command);
+
return result;
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch CASSANDRA-19877 created (now bf34e0b0)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch CASSANDRA-19877 in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git at bf34e0b0 Compaction This branch includes the following new commits: new bf34e0b0 Compaction The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 02/02: Fix SavedCommandTest
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git commit a33dea532956a697c8d87d1f4a275c8a48ad1023 Author: Alex Petrov AuthorDate: Mon Sep 16 18:21:00 2024 +0200 Fix SavedCommandTest --- src/java/org/apache/cassandra/service/accord/SavedCommand.java | 5 + test/unit/org/apache/cassandra/utils/AccordGenerators.java | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/service/accord/SavedCommand.java b/src/java/org/apache/cassandra/service/accord/SavedCommand.java index 5f5fb9b9c9..bef75a70ef 100644 --- a/src/java/org/apache/cassandra/service/accord/SavedCommand.java +++ b/src/java/org/apache/cassandra/service/accord/SavedCommand.java @@ -553,6 +553,11 @@ public class SavedCommand case PreAccepted: return Command.PreAccepted.preAccepted(attrs, executeAt, promised); case AcceptedInvalidate: +if (saveStatus == SaveStatus.AcceptedInvalidateWithDefinition) +return Command.Accepted.accepted(attrs, saveStatus, executeAt, promised, acceptedOrCommitted); +else +return Command.AcceptedInvalidateWithoutDefinition.acceptedInvalidate(attrs, promised, acceptedOrCommitted); + case Accepted: case PreCommitted: return Command.Accepted.accepted(attrs, saveStatus, executeAt, promised, acceptedOrCommitted); diff --git a/test/unit/org/apache/cassandra/utils/AccordGenerators.java b/test/unit/org/apache/cassandra/utils/AccordGenerators.java index 58042e2d5f..02715aa9cc 100644 --- a/test/unit/org/apache/cassandra/utils/AccordGenerators.java +++ b/test/unit/org/apache/cassandra/utils/AccordGenerators.java @@ -228,8 +228,10 @@ public class AccordGenerators return Command.SerializerSupport.notDefined(attributes(saveStatus), Ballot.ZERO); case PreAccepted: return Command.SerializerSupport.preaccepted(attributes(saveStatus), executeAt, Ballot.ZERO); -case Accepted: case AcceptedInvalidate: +return Command.AcceptedInvalidateWithoutDefinition.acceptedInvalidate(attributes(saveStatus), promised, Ballot.ZERO); + +case Accepted: case AcceptedWithDefinition: case AcceptedInvalidateWithDefinition: case PreCommittedWithDefinition: - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 01/02: Journal segment compaction
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit c831042910fce4b2ce1a92a0d86c2003d44b853f
Author: Alex Petrov
AuthorDate: Thu Aug 22 18:48:03 2024 +0200
Journal segment compaction
Patch by Alex Petrov and Aleksey Yeschenko, reviewed by Aleksey Yeschenko
and Alex Petrov for CASSANDRA-19876
---
.../org/apache/cassandra/config/AccordSpec.java| 14 ++
.../apache/cassandra/journal/ActiveSegment.java| 1 +
.../org/apache/cassandra/journal/Compactor.java| 102 +
.../org/apache/cassandra/journal/Descriptor.java | 13 +-
.../apache/cassandra/journal/EntrySerializer.java | 49 +++--
.../apache/cassandra/journal/InMemoryIndex.java| 4 +-
src/java/org/apache/cassandra/journal/Index.java | 13 +-
src/java/org/apache/cassandra/journal/Journal.java | 164 +++
.../org/apache/cassandra/journal/Metadata.java | 2 +-
.../org/apache/cassandra/journal/OnDiskIndex.java | 63 +++---
src/java/org/apache/cassandra/journal/Params.java | 4 +
.../apache/cassandra/journal/RecordConsumer.java | 1 +
src/java/org/apache/cassandra/journal/Segment.java | 11 +-
.../{RecordConsumer.java => SegmentCompactor.java} | 18 +-
.../org/apache/cassandra/journal/Segments.java | 42 ++--
.../apache/cassandra/journal/StaticSegment.java| 204 +-
.../cassandra/service/accord/AccordJournal.java| 29 +--
.../service/accord/AccordJournalTable.java | 227 +
.../cassandra/service/accord/AccordKeyspace.java | 18 +-
.../service/accord/AccordSegmentCompactor.java | 119 +++
.../cassandra/service/accord/IAccordService.java | 3 +
.../cassandra/service/accord/JournalKey.java | 6 +-
.../cassandra/service/accord/SavedCommand.java | 132 ++--
...Test.java => AccordJournalIntegrationTest.java} | 2 +-
.../journal/AccordJournalCompactionTest.java | 137 +
.../test/AccordJournalSimulationTest.java | 4 +-
.../org/apache/cassandra/journal/IndexTest.java| 12 +-
.../org/apache/cassandra/journal/JournalTest.java | 36 +++-
.../org/apache/cassandra/journal/SegmentTest.java | 10 +-
.../org/apache/cassandra/journal/TestParams.java | 12 ++
.../cassandra/service/accord/AccordTestUtils.java | 2 +-
31 files changed, 1160 insertions(+), 294 deletions(-)
diff --git a/src/java/org/apache/cassandra/config/AccordSpec.java
b/src/java/org/apache/cassandra/config/AccordSpec.java
index b4d25d6689..102ae68b67 100644
--- a/src/java/org/apache/cassandra/config/AccordSpec.java
+++ b/src/java/org/apache/cassandra/config/AccordSpec.java
@@ -28,6 +28,8 @@ public class AccordSpec
public volatile String journal_directory;
+public volatile boolean enable_journal_compaction = true;
+
public volatile OptionaldPositiveInt shard_count =
OptionaldPositiveInt.UNDEFINED;
public volatile DurationSpec.IntMillisecondsBound recover_delay = new
DurationSpec.IntMillisecondsBound(1000);
@@ -101,6 +103,18 @@ public class AccordSpec
return flushMode;
}
+@Override
+public boolean enableCompaction()
+{
+return DatabaseDescriptor.getAccord().enable_journal_compaction;
+}
+
+@Override
+public int compactionPeriodMillis()
+{
+return 60_000;
+}
+
@JsonIgnore
@Override
public int flushPeriodMillis()
diff --git a/src/java/org/apache/cassandra/journal/ActiveSegment.java
b/src/java/org/apache/cassandra/journal/ActiveSegment.java
index 1bee25a96f..1fd9905490 100644
--- a/src/java/org/apache/cassandra/journal/ActiveSegment.java
+++ b/src/java/org/apache/cassandra/journal/ActiveSegment.java
@@ -197,6 +197,7 @@ final class ActiveSegment extends Segment
descriptor.fileFor(Component.SYNCED_OFFSETS).deleteIfExists();
}
+@Override
void release()
{
selfRef.release();
diff --git a/src/java/org/apache/cassandra/journal/Compactor.java
b/src/java/org/apache/cassandra/journal/Compactor.java
new file mode 100644
index 00..846dd62ba8
--- /dev/null
+++ b/src/java/org/apache/cassandra/journal/Compactor.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS I
(cassandra) branch cep-15-accord updated (14546c96f5 -> a33dea5329)
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a change to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
from 14546c96f5 ninja: fix test build
new c831042910 Journal segment compaction
new a33dea5329 Fix SavedCommandTest
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/cassandra/config/AccordSpec.java| 14 ++
.../apache/cassandra/journal/ActiveSegment.java| 1 +
.../org/apache/cassandra/journal/Compactor.java| 102 +
.../org/apache/cassandra/journal/Descriptor.java | 13 +-
.../apache/cassandra/journal/EntrySerializer.java | 49 +++--
.../apache/cassandra/journal/InMemoryIndex.java| 4 +-
src/java/org/apache/cassandra/journal/Index.java | 13 +-
src/java/org/apache/cassandra/journal/Journal.java | 164 +++
.../org/apache/cassandra/journal/Metadata.java | 2 +-
.../org/apache/cassandra/journal/OnDiskIndex.java | 63 +++---
src/java/org/apache/cassandra/journal/Params.java | 4 +
.../apache/cassandra/journal/RecordConsumer.java | 1 +
src/java/org/apache/cassandra/journal/Segment.java | 11 +-
...JournalReadError.java => SegmentCompactor.java} | 20 +-
.../org/apache/cassandra/journal/Segments.java | 42 ++--
.../apache/cassandra/journal/StaticSegment.java| 204 +-
.../cassandra/service/accord/AccordJournal.java| 29 +--
.../service/accord/AccordJournalTable.java | 227 +
.../cassandra/service/accord/AccordKeyspace.java | 18 +-
.../service/accord/AccordSegmentCompactor.java | 119 +++
.../cassandra/service/accord/IAccordService.java | 3 +
.../cassandra/service/accord/JournalKey.java | 6 +-
.../cassandra/service/accord/SavedCommand.java | 137 +++--
...Test.java => AccordJournalIntegrationTest.java} | 2 +-
.../journal/AccordJournalCompactionTest.java | 137 +
.../test/AccordJournalSimulationTest.java | 4 +-
.../org/apache/cassandra/journal/IndexTest.java| 12 +-
.../org/apache/cassandra/journal/JournalTest.java | 36 +++-
.../org/apache/cassandra/journal/SegmentTest.java | 10 +-
.../org/apache/cassandra/journal/TestParams.java | 12 ++
.../cassandra/service/accord/AccordTestUtils.java | 2 +-
.../apache/cassandra/utils/AccordGenerators.java | 4 +-
32 files changed, 1164 insertions(+), 301 deletions(-)
create mode 100644 src/java/org/apache/cassandra/journal/Compactor.java
copy src/java/org/apache/cassandra/journal/{JournalReadError.java =>
SegmentCompactor.java} (65%)
create mode 100644
src/java/org/apache/cassandra/service/accord/AccordJournalTable.java
create mode 100644
src/java/org/apache/cassandra/service/accord/AccordSegmentCompactor.java
rename
test/distributed/org/apache/cassandra/distributed/test/accord/{AccordJournalTest.java
=> AccordJournalIntegrationTest.java} (98%)
create mode 100644
test/distributed/org/apache/cassandra/journal/AccordJournalCompactionTest.java
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cep-15-accord updated: Fix Journal segment allocation/switch race condition
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cep-15-accord by this push:
new fb7e29dd42 Fix Journal segment allocation/switch race condition
fb7e29dd42 is described below
commit fb7e29dd4298a5ecf93a3c2404ce0f4409e56627
Author: Alex Petrov
AuthorDate: Mon Sep 9 15:05:32 2024 +0200
Fix Journal segment allocation/switch race condition
Patch by Alex Petrov; reviewed by Aleksey Yeschenko for CASSANDRA-19866
---
src/java/org/apache/cassandra/journal/Journal.java | 4
1 file changed, 4 insertions(+)
diff --git a/src/java/org/apache/cassandra/journal/Journal.java
b/src/java/org/apache/cassandra/journal/Journal.java
index c092363af7..50a3058ec9 100644
--- a/src/java/org/apache/cassandra/journal/Journal.java
+++ b/src/java/org/apache/cassandra/journal/Journal.java
@@ -198,6 +198,10 @@ public class Journal implements Shutdownable
public boolean isFlushed(RecordPointer recordPointer)
{
+Segment current = currentSegment;
+if (current.descriptor.timestamp == recordPointer.segment)
+return current.isFlushed(recordPointer.position);
+
return segments.get().isFlushed(recordPointer);
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cep-15-accord updated (c9be786ded -> f6ef84ce2d)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git from c9be786ded Create a fuzz test that randomizes topology changes, cluster actions, and CQL operations add f6ef84ce2d Fix a problem with static segments being opened with an empty offset after switch unless active segments offset file was closed No new revisions were added by this update. Summary of changes: src/java/org/apache/cassandra/journal/ActiveSegment.java | 1 + src/java/org/apache/cassandra/journal/SyncedOffsets.java | 1 + 2 files changed, 2 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cep-15-accord updated: Switch to streaming serialization of SavedCommand
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cep-15-accord by this push:
new 059ce88143 Switch to streaming serialization of SavedCommand
059ce88143 is described below
commit 059ce88143b55e3345bc49c69b20a9fb62d27567
Author: Alex Petrov
AuthorDate: Wed Aug 28 17:18:36 2024 +0200
Switch to streaming serialization of SavedCommand
Patch by Alex Petrov; reviewed by David Capwell for CASSANDRA-19865
Co-authored-by: dcapwell
---
modules/accord | 2 +-
src/java/org/apache/cassandra/journal/Journal.java | 44 +-
.../service/accord/AccordCommandStore.java | 6 +-
.../cassandra/service/accord/AccordJournal.java| 360 ++---
.../cassandra/service/accord/AccordKeyspace.java | 3 +
.../service/accord/AccordObjectSizes.java | 2 +-
.../service/accord/AccordSafeCommand.java | 2 +-
.../apache/cassandra/service/accord/IJournal.java | 8 +-
.../cassandra/service/accord/JournalKey.java | 37 +-
.../cassandra/service/accord/SavedCommand.java | 833 ++---
.../service/accord/async/AsyncOperation.java | 27 +-
.../accord/serializers/WaitingOnSerializer.java| 4 +-
.../compaction/CompactionAccordIteratorsTest.java | 4 +-
.../service/accord/AccordJournalOrderTest.java | 22 +-
.../service/accord/AccordJournalTest.java | 3 +-
.../cassandra/service/accord/AccordTestUtils.java | 14 +-
.../cassandra/service/accord/MockJournal.java | 324 +++-
.../cassandra/service/accord/SavedCommandTest.java | 144
.../service/accord/async/AsyncOperationTest.java | 15 +
.../serializers/CommandsForKeySerializerTest.java | 4 +-
.../serializers/WaitingOnSerializerTest.java | 2 +-
.../apache/cassandra/utils/AccordGenerators.java | 172 -
22 files changed, 1159 insertions(+), 873 deletions(-)
diff --git a/modules/accord b/modules/accord
index 178952b41f..449b2b4d0b 16
--- a/modules/accord
+++ b/modules/accord
@@ -1 +1 @@
-Subproject commit 178952b41f05bfa307aef03dcc013e37fb6230b4
+Subproject commit 449b2b4d0bf4bb44d55a3c57f712a4d5a15e7220
diff --git a/src/java/org/apache/cassandra/journal/Journal.java
b/src/java/org/apache/cassandra/journal/Journal.java
index db86106d42..c092363af7 100644
--- a/src/java/org/apache/cassandra/journal/Journal.java
+++ b/src/java/org/apache/cassandra/journal/Journal.java
@@ -47,11 +47,13 @@ import
org.apache.cassandra.concurrent.SequentialExecutorPlus;
import org.apache.cassandra.concurrent.Shutdownable;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.PathUtils;
import org.apache.cassandra.journal.Segments.ReferencedSegment;
import org.apache.cassandra.journal.Segments.ReferencedSegments;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.SavedCommand;
import org.apache.cassandra.utils.Crc;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.Simulate;
@@ -343,11 +345,16 @@ public class Journal implements Shutdownable
return null;
}
-// TODO: This should be improved with new index that should take better
care of handling multiple items
public List readAll(K id)
{
-EntrySerializer.EntryHolder holder = new
EntrySerializer.EntryHolder<>();
List res = new ArrayList<>(2);
+readAll(id, (in, userVersion) ->
res.add(valueSerializer.deserialize(id, in, userVersion)));
+return res;
+}
+
+public void readAll(K id, Reader reader)
+{
+EntrySerializer.EntryHolder holder = new
EntrySerializer.EntryHolder<>();
try (ReferencedSegments segments = selectAndReference(id))
{
for (Segment segment : segments.all())
@@ -357,7 +364,7 @@ public class Journal implements Shutdownable
{
Invariants.checkState(Objects.equals(holder.key, id),
"%s != %s", holder.key, id);
-res.add(valueSerializer.deserialize(holder.key, in,
segment.descriptor.userVersion));
+reader.read(in, segment.descriptor.userVersion);
holder.clear();
}
catch (IOException e)
@@ -368,7 +375,6 @@ public class Journal implements Shutdownable
});
}
}
-return res;
}
/**
@@ -504,11 +510,28 @@ public class Journal implements Shutdownable
* @param hosts hosts expected to invalidate
(cassandra-accord) branch trunk updated: Fix CompactionIteratorTest, switch to streaming serialization of SavedCommand
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
The following commit(s) were added to refs/heads/trunk by this push:
new 449b2b4d Fix CompactionIteratorTest, switch to streaming serialization
of SavedCommand
449b2b4d is described below
commit 449b2b4d0bf4bb44d55a3c57f712a4d5a15e7220
Author: Alex Petrov
AuthorDate: Wed Aug 28 12:01:53 2024 +0200
Fix CompactionIteratorTest, switch to streaming serialization of
SavedCommand
Patch by Alex Petrov; reviewed by David Capwell for CASSANDRA-19865
Co-authored-by: dcapwell
---
.../src/main/java/accord/local/Command.java| 18 ++--
.../src/main/java/accord/local/Commands.java | 2 +-
.../main/java/accord/utils/ImmutableBitSet.java| 8
.../test/java/accord/messages/ReadDataTest.java| 2 +-
.../src/test/java/accord/utils/AccordGens.java | 51 +-
.../test/java/accord/utils/ReflectionUtils.java| 6 +++
.../src/main/java/accord/maelstrom/Packet.java | 1 +
7 files changed, 80 insertions(+), 8 deletions(-)
diff --git a/accord-core/src/main/java/accord/local/Command.java
b/accord-core/src/main/java/accord/local/Command.java
index 8c98882d..1a886c35 100644
--- a/accord-core/src/main/java/accord/local/Command.java
+++ b/accord-core/src/main/java/accord/local/Command.java
@@ -39,6 +39,7 @@ import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
import accord.primitives.RangeDeps;
import accord.primitives.Ranges;
+import accord.primitives.Routable;
import accord.primitives.Route;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
@@ -1223,7 +1224,15 @@ public abstract class Command implements CommonAttributes
public static class WaitingOn
{
-public static final WaitingOn EMPTY = new WaitingOn(Keys.EMPTY,
RangeDeps.NONE, KeyDeps.NONE, ImmutableBitSet.EMPTY, ImmutableBitSet.EMPTY);
+private static final WaitingOn EMPTY_FOR_KEY = new
WaitingOn(Keys.EMPTY, RangeDeps.NONE, KeyDeps.NONE, ImmutableBitSet.EMPTY,
null);
+private static final WaitingOn EMPTY_FOR_RANGE = new
WaitingOn(Keys.EMPTY, RangeDeps.NONE, KeyDeps.NONE, ImmutableBitSet.EMPTY,
ImmutableBitSet.EMPTY);
+
+public static WaitingOn empty(Routable.Domain domain)
+{
+if (domain == Range)
+return EMPTY_FOR_RANGE;
+return EMPTY_FOR_KEY;
+}
public final Keys keys;
public final RangeDeps directRangeDeps;
@@ -1288,10 +1297,11 @@ public abstract class Command implements
CommonAttributes
return ifNull;
}
-public static WaitingOn none(Deps deps)
+public static WaitingOn none(Routable.Domain domain, Deps deps)
{
-ImmutableBitSet empty = new
ImmutableBitSet(deps.rangeDeps.txnIdCount() + deps.directKeyDeps.txnIdCount() +
deps.keyDeps.keys().size());
-return new WaitingOn(deps.keyDeps.keys(), deps.rangeDeps,
deps.directKeyDeps, empty, empty);
+return new WaitingOn(deps.keyDeps.keys(), deps.rangeDeps,
deps.directKeyDeps,
+ new
ImmutableBitSet(deps.directKeyDeps.txnIdCount() + deps.keyDeps.keys().size()),
+ domain == Range ? new
ImmutableBitSet(deps.rangeDeps.txnIdCount()) : null);
}
public boolean isWaiting()
diff --git a/accord-core/src/main/java/accord/local/Commands.java
b/accord-core/src/main/java/accord/local/Commands.java
index d3387a6b..bb8c8ba4 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -395,7 +395,7 @@ public class Commands
PartialTxn partialTxn = emptyTxn.slice(coordinateRanges, true);
Invariants.checkState(validate(SaveStatus.Stable, command,
coordinateRanges, route, partialTxn, none, null));
CommonAttributes newAttributes = set(SaveStatus.Stable, command,
command, coordinateRanges, Ballot.ZERO, route, partialTxn, none);
-safeCommand.stable(safeStore, newAttributes, Ballot.ZERO, localSyncId,
WaitingOn.EMPTY);
+safeCommand.stable(safeStore, newAttributes, Ballot.ZERO, localSyncId,
WaitingOn.empty(emptyTxn.keys().domain()));
safeStore.notifyListeners(safeCommand);
}
diff --git a/accord-core/src/main/java/accord/utils/ImmutableBitSet.java
b/accord-core/src/main/java/accord/utils/ImmutableBitSet.java
index 281c0d16..ef8a33a7 100644
--- a/accord-core/src/main/java/accord/utils/ImmutableBitSet.java
+++ b/accord-core/src/main/java/accord/utils/ImmutableBitSet.java
@@ -77,4 +77,12 @@ public class ImmutableBitSet extends SimpleBitSet
{
throw new UnsupportedOperationException();
}
+
+@Override
+public String toString()
+{
+return "ImmutableBitSet{" +
+
(cassandra) branch cep-15-accord updated (fd1d6a437b -> 52a9db8eb5)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git from fd1d6a437b Fix flusher shutdown; ignore repair test add 52a9db8eb5 Remove ignore No new revisions were added by this update. Summary of changes: .../cassandra/distributed/test/accord/AccordIncrementalRepairTest.java | 2 -- 1 file changed, 2 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch CASSANDRA-19865 updated (61088d1e -> 26d67917)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch CASSANDRA-19865 in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git omit 61088d1e Fix CompactionIteratorTest add 178952b4 add .asf.yaml add 55bd05a0 Fix CompactionIteratorTest add 55e67f5a added more generators add 2f290347 mark diff if the classes dont match add 89524631 Draw distinction between Waiting on for keys and ranges add 26d67917 Fix SimpleRandomTest after HEAD~1 commit This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (61088d1e) \ N -- N -- N refs/heads/CASSANDRA-19865 (26d67917) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .asf.yaml | 36 +++ .../src/main/java/accord/local/Command.java| 15 +-- .../src/main/java/accord/local/Commands.java | 2 +- .../main/java/accord/utils/ImmutableBitSet.java| 8 .../test/java/accord/messages/ReadDataTest.java| 2 +- .../src/test/java/accord/utils/AccordGens.java | 51 +- .../test/java/accord/utils/ReflectionUtils.java| 6 +++ .../src/main/java/accord/maelstrom/Packet.java | 1 + 8 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 .asf.yaml - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cep-15-accord updated: Fix flusher shutdown; ignore repair test
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cep-15-accord by this push:
new fd1d6a437b Fix flusher shutdown; ignore repair test
fd1d6a437b is described below
commit fd1d6a437b85e41f8e35f82c4da45a38671add96
Author: Alex Petrov
AuthorDate: Wed Aug 28 17:18:36 2024 +0200
Fix flusher shutdown; ignore repair test
---
src/java/org/apache/cassandra/journal/Flusher.java | 2 +-
.../distributed/test/accord/AccordIncrementalRepairTest.java | 5 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/src/java/org/apache/cassandra/journal/Flusher.java
b/src/java/org/apache/cassandra/journal/Flusher.java
index d95573d1ad..709c68615b 100644
--- a/src/java/org/apache/cassandra/journal/Flusher.java
+++ b/src/java/org/apache/cassandra/journal/Flusher.java
@@ -105,7 +105,7 @@ final class Flusher
flushExecutor.awaitTermination(1, MINUTES);
if (fsyncExecutor != null)
{
-fsyncExecutor.shutdown();
+fsyncExecutor.shutdownNow(); // `now` to interrupt potentially
parked runnable
fsyncExecutor.awaitTermination(1, MINUTES);
}
}
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java
b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java
index 452d9dc53a..f6193c74f8 100644
---
a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java
+++
b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java
@@ -32,6 +32,7 @@ import com.google.common.collect.Iterables;
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -248,6 +249,10 @@ public class AccordIncrementalRepairTest extends
AccordTestBase
return txnId;
}
+// TODO (required): After conversation with Ariel: it's a known issue that
I am not sure we need to fix now.
+// The problem is that we don't flush after Accord repair, but before
data repair when running incremental
+// repair so it doesn't see the repaired sstables it is checking for.
+@Ignore
@Test
public void txnRepairTest() throws Throwable
{
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch trunk updated: Fix CompactionIteratorTest
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
The following commit(s) were added to refs/heads/trunk by this push:
new 55bd05a0 Fix CompactionIteratorTest
55bd05a0 is described below
commit 55bd05a0612acbec29a1bf2ee0cb11d5ef754200
Author: Alex Petrov
AuthorDate: Wed Aug 28 12:01:53 2024 +0200
Fix CompactionIteratorTest
Patch by Alex Petrov; reviewed by David Capwell for CASSANDRA-19865
---
accord-core/src/main/java/accord/primitives/TxnId.java | 8
1 file changed, 8 insertions(+)
diff --git a/accord-core/src/main/java/accord/primitives/TxnId.java
b/accord-core/src/main/java/accord/primitives/TxnId.java
index 63824421..8c3e9c5f 100644
--- a/accord-core/src/main/java/accord/primitives/TxnId.java
+++ b/accord-core/src/main/java/accord/primitives/TxnId.java
@@ -21,6 +21,8 @@ package accord.primitives;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.annotations.VisibleForTesting;
+
import accord.local.Node.Id;
import accord.primitives.Routable.Domain;
import accord.primitives.Txn.Kind;
@@ -104,6 +106,12 @@ public class TxnId extends Timestamp
return new TxnId(epoch(), hlc(), kind, domain(), node);
}
+@VisibleForTesting
+public TxnId as(Kind kind, Domain domain)
+{
+return new TxnId(epoch(), hlc(), kind, domain, node);
+}
+
public TxnId withEpoch(long epoch)
{
return epoch == epoch() ? this : new TxnId(epoch, hlc(), flags(),
node);
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cep-15-accord updated (c1029bf02d -> 27d74c7f85)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git omit c1029bf02d Add native_transport_timeout to this branch to reduce test failure noise new 63bb9a0c7d Add native_transport_timeout to this branch to reduce test failure noise new 27d74c7f85 Cleanup; repin Accord This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (c1029bf02d) \ N -- N -- N refs/heads/cep-15-accord (27d74c7f85) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: modules/accord | 2 +- src/java/org/apache/cassandra/config/Config.java | 2 +- src/java/org/apache/cassandra/journal/Flusher.java | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 01/02: Add native_transport_timeout to this branch to reduce test failure noise
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 63bb9a0c7d092132c5cc007b25c892b10710b434
Author: Alex Petrov
AuthorDate: Wed Aug 28 17:18:36 2024 +0200
Add native_transport_timeout to this branch to reduce test failure noise
---
src/java/org/apache/cassandra/config/Config.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/java/org/apache/cassandra/config/Config.java
b/src/java/org/apache/cassandra/config/Config.java
index 1ac92db22d..0d12b92664 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -1326,4 +1326,7 @@ public class Config
}
public TriggersPolicy triggers_policy = TriggersPolicy.enabled;
+
+// TODO (required): this is only here to avoid test python dtest failures;
remove after trunk rebase
+public DurationSpec.LongMillisecondsBound native_transport_timeout = new
DurationSpec.LongMillisecondsBound("12s");
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 02/02: Cleanup; repin Accord
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git commit 27d74c7f851f41bcbecfa5112be5ca1cac5e162c Author: Alex Petrov AuthorDate: Wed Aug 28 17:36:05 2024 +0200 Cleanup; repin Accord --- modules/accord | 2 +- src/java/org/apache/cassandra/journal/Flusher.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/accord b/modules/accord index 129a4862df..178952b41f 16 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 129a4862df43fdc5893687922a77bb0288f8cb83 +Subproject commit 178952b41f05bfa307aef03dcc013e37fb6230b4 diff --git a/src/java/org/apache/cassandra/journal/Flusher.java b/src/java/org/apache/cassandra/journal/Flusher.java index ee52aa307d..d95573d1ad 100644 --- a/src/java/org/apache/cassandra/journal/Flusher.java +++ b/src/java/org/apache/cassandra/journal/Flusher.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.journal; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; @@ -529,4 +528,4 @@ final class Flusher void onFlushFailed(Throwable cause); } -} \ No newline at end of file +} - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 01/04: Avoid ClassCastException when verifying tables with reversed partitioner
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 139f324969a564d32bd88e36a7abbc7d2bbbc8f0
Author: Marcus Eriksson
AuthorDate: Thu Apr 25 19:44:39 2024 +0200
Avoid ClassCastException when verifying tables with reversed partitioner
Patch by Marcus Eriksson; reviewed by Sam Tunnicliffe for
CASSANDRA-19710
---
.../cassandra/db/compaction/CompactionManager.java | 6 --
.../io/sstable/format/SortedTableVerifier.java | 4 +++-
.../org/apache/cassandra/io/sstable/VerifyTest.java| 18 ++
3 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 6899d58c60..edf6df5c99 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -99,6 +99,7 @@ import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.locator.RangesAtEndpoint;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.metrics.CompactionMetrics;
@@ -628,8 +629,9 @@ public class CompactionManager implements
CompactionManagerMBean, ICompactionMan
logger.info("Cleanup cannot run before a node has joined the
ring");
return AllSSTableOpStatus.ABORTED;
}
-if (cfStore.keyspace.getMetadata().params.replication.isMeta())
-return AllSSTableOpStatus.SUCCESSFUL; // todo - we probably want
to be able to cleanup MetaStrategy keyspaces
+if (cfStore.getPartitioner() == MetaStrategy.partitioner)
+return AllSSTableOpStatus.SUCCESSFUL; // todo - we probably want
to be able to cleanup MetaStrategy keyspaces. When we fix this, also fix
+ //
SortedTableVerifier to make sure system_cluster_metadata is empty for non-CMS
instances
final boolean hasIndexes = cfStore.indexManager.hasIndexes();
// if local ranges is empty, it means no data should remain
diff --git
a/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
index f68e1c9684..eb6231f9e7 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
@@ -56,6 +56,7 @@ import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.util.DataIntegrityMetadata;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -152,7 +153,8 @@ public abstract class SortedTableVerifier imp
verifyBloomFilter();
-if (options.checkOwnsTokens && !isOffline && !(cfs.getPartitioner()
instanceof LocalPartitioner))
+// TODO: when making it possible to clean up system_cluster_metadata,
we should make sure that non-cms members don't have any sstables there
+if (options.checkOwnsTokens && !isOffline && !(cfs.getPartitioner()
instanceof LocalPartitioner) && !(cfs.getPartitioner() ==
MetaStrategy.partitioner))
{
if (verifyOwnedRanges() == 0)
return;
diff --git a/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
index c4a8a07aac..5493356a2e 100644
--- a/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
@@ -43,6 +43,7 @@ import org.apache.cassandra.batchlog.Batch;
import org.apache.cassandra.batchlog.BatchlogManager;
import org.apache.cassandra.cache.ChunkCache;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.db.BufferDecoratedKey;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
@@ -763,6 +764,23 @@ public class VerifyTest
}
}
+@Test
+public void testVerifyReversedPartitioner()
+{
+for (long i = 0; i < 10; i++)
+QueryProcessor.executeInternal("insert into
system.local_metadata_log (epoch) values (?)", i);
+ColumnFamilyStore cfs
(cassandra) 03/04: Wait for flusher shutdown
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit f21e038263448d8199c1b77919ff5fc0f189c7eb
Author: Alex Petrov
AuthorDate: Wed Aug 28 16:38:00 2024 +0200
Wait for flusher shutdown
---
src/java/org/apache/cassandra/journal/Flusher.java | 7 ++-
src/java/org/apache/cassandra/journal/Journal.java | 23 +++---
2 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/src/java/org/apache/cassandra/journal/Flusher.java
b/src/java/org/apache/cassandra/journal/Flusher.java
index 41cdd389e8..ee52aa307d 100644
--- a/src/java/org/apache/cassandra/journal/Flusher.java
+++ b/src/java/org/apache/cassandra/journal/Flusher.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.journal;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.LockSupport;
@@ -99,11 +100,15 @@ final class Flusher
flushExecutor = executorFactory().infiniteLoop(flushExecutorName, new
FlushRunnable(preciseTime), SAFE, NON_DAEMON, SYNCHRONIZED);
}
-void shutdown()
+void shutdown() throws InterruptedException
{
flushExecutor.shutdown();
+flushExecutor.awaitTermination(1, MINUTES);
if (fsyncExecutor != null)
+{
fsyncExecutor.shutdown();
+fsyncExecutor.awaitTermination(1, MINUTES);
+}
}
@Simulate(with={MONITORS,GLOBAL_CLOCK,LOCK_SUPPORT})
diff --git a/src/java/org/apache/cassandra/journal/Journal.java
b/src/java/org/apache/cassandra/journal/Journal.java
index d633956e51..db86106d42 100644
--- a/src/java/org/apache/cassandra/journal/Journal.java
+++ b/src/java/org/apache/cassandra/journal/Journal.java
@@ -244,13 +244,22 @@ public class Journal implements Shutdownable
public void shutdown()
{
-allocator.shutdown();
-//compactor.stop();
-//invalidator.stop();
-flusher.shutdown();
-closer.shutdown();
-closeAllSegments();
-metrics.deregister();
+try
+{
+allocator.shutdown();
+allocator.awaitTermination(1, TimeUnit.MINUTES);
+//compactor.stop();
+//invalidator.stop();
+flusher.shutdown();
+closer.shutdown();
+closer.awaitTermination(1, TimeUnit.MINUTES);
+closeAllSegments();
+metrics.deregister();
+}
+catch (InterruptedException e)
+{
+logger.error("Could not shutdown journal", e);
+}
}
@Override
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 04/04: Add native_transport_timeout to this branch to reduce test failure noise
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit c1029bf02da0a9fd9935ce91cf5dbb7a84a42b59
Author: Alex Petrov
AuthorDate: Wed Aug 28 17:18:36 2024 +0200
Add native_transport_timeout to this branch to reduce test failure noise
---
src/java/org/apache/cassandra/config/Config.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/java/org/apache/cassandra/config/Config.java
b/src/java/org/apache/cassandra/config/Config.java
index 1ac92db22d..b6b68504d2 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -1326,4 +1326,7 @@ public class Config
}
public TriggersPolicy triggers_policy = TriggersPolicy.enabled;
+
+// TODO (required): this is only here to avoid test python dtest failures
+public DurationSpec.LongMillisecondsBound native_transport_timeout = new
DurationSpec.LongMillisecondsBound("12s");
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 02/04: Add size to the segment index for safer journal reads
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit e9baf3e01799f4e026b1fad6543c2825310ecdca
Author: Alex Petrov
AuthorDate: Mon Jul 22 10:11:42 2024 +0200
Add size to the segment index for safer journal reads
Patch by Alex Petrov; reviewed by Marcus Eriksson for CASSANDRA-19871
---
.../apache/cassandra/journal/ActiveSegment.java| 22 ++--
.../apache/cassandra/journal/InMemoryIndex.java| 58 +-
src/java/org/apache/cassandra/journal/Index.java | 50 -
src/java/org/apache/cassandra/journal/Journal.java | 12 ++-
.../org/apache/cassandra/journal/OnDiskIndex.java | 115 +++-
src/java/org/apache/cassandra/journal/Params.java | 5 +
src/java/org/apache/cassandra/journal/Segment.java | 20 ++--
.../apache/cassandra/journal/SegmentWriter.java| 5 +-
.../apache/cassandra/journal/StaticSegment.java| 4 +-
.../org/apache/cassandra/journal/IndexTest.java| 120 ++---
10 files changed, 291 insertions(+), 120 deletions(-)
diff --git a/src/java/org/apache/cassandra/journal/ActiveSegment.java
b/src/java/org/apache/cassandra/journal/ActiveSegment.java
index ebbd672b80..a815d23199 100644
--- a/src/java/org/apache/cassandra/journal/ActiveSegment.java
+++ b/src/java/org/apache/cassandra/journal/ActiveSegment.java
@@ -125,9 +125,9 @@ final class ActiveSegment extends Segment
* Expects the caller to acquire the ref to the segment and the record to
exist.
*/
@Override
-boolean read(int offset, EntrySerializer.EntryHolder into)
+boolean read(int offset, int size, EntrySerializer.EntryHolder into)
{
-ByteBuffer duplicate =
buffer.duplicate().position(offset).limit(buffer.capacity());
+ByteBuffer duplicate =
buffer.duplicate().position(offset).limit(offset + size);
try
{
EntrySerializer.read(into, keySupport, duplicate,
descriptor.userVersion);
@@ -394,7 +394,7 @@ final class ActiveSegment extends Segment
opGroup.close();
return null;
}
-return new Allocation(opGroup,
buffer.duplicate().position(position).limit(position + totalSize));
+return new Allocation(opGroup,
buffer.duplicate().position(position).limit(position + totalSize), totalSize);
}
catch (Throwable t)
{
@@ -431,13 +431,15 @@ final class ActiveSegment extends Segment
{
private final OpOrder.Group appendOp;
private final ByteBuffer buffer;
-private final int position;
+private final int start;
+private final int length;
-Allocation(OpOrder.Group appendOp, ByteBuffer buffer)
+Allocation(OpOrder.Group appendOp, ByteBuffer buffer, int length)
{
this.appendOp = appendOp;
this.buffer = buffer;
-this.position = buffer.position();
+this.start = buffer.position();
+this.length = length;
}
RecordPointer write(K id, ByteBuffer record, Set hosts)
@@ -445,9 +447,9 @@ final class ActiveSegment extends Segment
try (BufferedDataOutputStreamPlus out = new
DataOutputBufferFixed(buffer))
{
EntrySerializer.write(id, record, hosts, keySupport, out,
descriptor.userVersion);
-index.update(id, position);
+index.update(id, start, length);
metadata.update(hosts);
-return new RecordPointer(descriptor.timestamp, position);
+return new RecordPointer(descriptor.timestamp, start);
}
catch (IOException e)
{
@@ -465,7 +467,7 @@ final class ActiveSegment extends Segment
try (BufferedDataOutputStreamPlus out = new
DataOutputBufferFixed(buffer))
{
EntrySerializer.write(id, record, hosts, keySupport, out,
descriptor.userVersion);
-index.update(id, position);
+index.update(id, start, length);
metadata.update(hosts);
}
catch (IOException e)
@@ -482,7 +484,7 @@ final class ActiveSegment extends Segment
{
try (Timer.Context ignored = waitingOnFlush.time())
{
-waitForFlush(position);
+waitForFlush(start);
}
}
}
diff --git a/src/java/org/apache/cassandra/journal/InMemoryIndex.java
b/src/java/org/apache/cassandra/journal/InMemoryIndex.java
index 1ff4a28d7a..5417bfea40 100644
--- a/src/java/org/apache/cassandra/journal/InMemoryIndex.java
+++ b/src/java/org/apache/cassandra/journal/InMemoryIndex.java
@@ -34,9 +34,9 @@ import org.apache.cassandra.io.util.FileOutputStreamPlus;
*/
final class InMemoryIndex extends Index
{
-private static final int[] EMPTY = new int[0
(cassandra) branch cep-15-accord updated (e2583bbdbf -> c1029bf02d)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git from e2583bbdbf Switch to infinite loop executor instead of a while-loop thread. new 139f324969 Avoid ClassCastException when verifying tables with reversed partitioner new e9baf3e017 Add size to the segment index for safer journal reads new f21e038263 Wait for flusher shutdown new c1029bf02d Add native_transport_timeout to this branch to reduce test failure noise The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/java/org/apache/cassandra/config/Config.java | 3 + .../cassandra/db/compaction/CompactionManager.java | 6 +- .../io/sstable/format/SortedTableVerifier.java | 4 +- .../apache/cassandra/journal/ActiveSegment.java| 22 ++-- src/java/org/apache/cassandra/journal/Flusher.java | 7 +- .../apache/cassandra/journal/InMemoryIndex.java| 58 +- src/java/org/apache/cassandra/journal/Index.java | 50 - src/java/org/apache/cassandra/journal/Journal.java | 35 +++--- .../org/apache/cassandra/journal/OnDiskIndex.java | 115 +++- src/java/org/apache/cassandra/journal/Params.java | 5 + src/java/org/apache/cassandra/journal/Segment.java | 20 ++-- .../apache/cassandra/journal/SegmentWriter.java| 5 +- .../apache/cassandra/journal/StaticSegment.java| 4 +- .../apache/cassandra/io/sstable/VerifyTest.java| 18 .../org/apache/cassandra/journal/IndexTest.java| 120 ++--- 15 files changed, 341 insertions(+), 131 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 01/02: Revert acccord module to absolute path
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git commit 5defc358ddface6d718e6447f5189c156d00acbd Author: Alex Petrov AuthorDate: Mon Aug 26 12:54:54 2024 +0200 Revert acccord module to absolute path --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 60a9510e7a..616dacf610 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "modules/accord"] path = modules/accord - url = ../cassandra-accord.git + url = https://github.com/apache/cassandra-accord.git branch = trunk - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 02/02: Switch to infinite loop executor instead of a while-loop thread.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit e2583bbdbf3f13c476bc32baf06deccf9d12fe58
Author: Alex Petrov
AuthorDate: Mon Aug 26 12:51:29 2024 +0200
Switch to infinite loop executor instead of a while-loop thread.
Patch by Alex Petrov; reviewed by David Capwell for CASSANDRA-19864
---
.../cassandra/service/accord/AccordJournal.java| 142 -
1 file changed, 83 insertions(+), 59 deletions(-)
diff --git a/src/java/org/apache/cassandra/service/accord/AccordJournal.java
b/src/java/org/apache/cassandra/service/accord/AccordJournal.java
index 956473cfd8..cf4fab6e16 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordJournal.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordJournal.java
@@ -52,6 +52,8 @@ import accord.primitives.TxnId;
import accord.utils.Invariants;
import org.agrona.collections.Long2ObjectHashMap;
import org.agrona.collections.LongArrayList;
+import org.apache.cassandra.concurrent.InfiniteLoopExecutor;
+import org.apache.cassandra.concurrent.Interruptible;
import org.apache.cassandra.concurrent.ManyToOneConcurrentLinkedQueue;
import org.apache.cassandra.concurrent.Shutdownable;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -105,6 +107,9 @@ import static
accord.messages.MessageType.SET_SHARD_DURABLE_REQ;
import static accord.messages.MessageType.STABLE_FAST_PATH_REQ;
import static accord.messages.MessageType.STABLE_MAXIMAL_REQ;
import static accord.messages.MessageType.STABLE_SLOW_PATH_REQ;
+import static
org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static
org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+import static org.apache.cassandra.concurrent.Interruptible.State.NORMAL;
import static
org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType.INTEROP_APPLY_MAXIMAL_REQ;
import static
org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType.INTEROP_APPLY_MINIMAL_REQ;
import static
org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType.INTEROP_COMMIT_MAXIMAL_REQ;
@@ -165,7 +170,7 @@ public class AccordJournal implements IJournal, Shutdownable
{
Invariants.checkState(status == Status.STARTED);
status = Status.TERMINATING;
-delayedRequestProcessor.runOnce();
+delayedRequestProcessor.shutdown();
journal.shutdown();
status = Status.TERMINATED;
}
@@ -625,12 +630,18 @@ public class AccordJournal implements IJournal,
Shutdownable
* Handling topology changes / epoch shift
*/
-private final class DelayedRequestProcessor extends Thread
+private class DelayedRequestProcessor implements Interruptible.Task
{
private final ManyToOneConcurrentLinkedQueue
delayedRequests = new ManyToOneConcurrentLinkedQueue<>();
private final LongArrayList waitForEpochs = new LongArrayList();
private final Long2ObjectHashMap> byEpoch = new
Long2ObjectHashMap<>();
private final AtomicReference signal = new
AtomicReference<>(Condition.newOneTimeCondition());
+private volatile Interruptible executor;
+
+public void start()
+{
+ executor =
executorFactory().infiniteLoop("AccordJournal-delayed-request-processor",
this::run, SAFE, InfiniteLoopExecutor.Daemon.NON_DAEMON,
InfiniteLoopExecutor.Interrupts.SYNCHRONIZED);
+}
private void delay(RequestContext requestContext)
{
@@ -643,81 +654,94 @@ public class AccordJournal implements IJournal,
Shutdownable
signal.get().signal();
}
-public void run()
+@Override
+public void run(Interruptible.State state)
{
-while (!Thread.currentThread().isInterrupted() &&
isRunnable(status))
+if (state != NORMAL || Thread.currentThread().isInterrupted() ||
!isRunnable(status))
+return;
+
+try
{
-try
+Condition signal = Condition.newOneTimeCondition();
+this.signal.set(signal);
+// First, poll delayed requests, put them into by epoch
+while (!delayedRequests.isEmpty())
{
-Condition signal = Condition.newOneTimeCondition();
-this.signal.set(signal);
-// First, poll delayed requests, put them into by epoch
-while (!delayedRequests.isEmpty())
+RequestContext context = delayedRequests.poll();
+long waitForEpoch = context.waitForEpoch;
+
+List l =
byEpoch.computeIfAbsent(waitForEpoch, (ignore) -> new ArrayList<>());
+if (l.isEmpty())
+
(cassandra) branch cep-15-accord updated (bfadfdc123 -> e2583bbdbf)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git omit bfadfdc123 Revert acccord module to absolute path omit 6b2bd52447 Switch to infinite loop executor instead of a while-loop thread. new 5defc358dd Revert acccord module to absolute path new e2583bbdbf Switch to infinite loop executor instead of a while-loop thread. This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (bfadfdc123) \ N -- N -- N refs/heads/cep-15-accord (e2583bbdbf) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/java/org/apache/cassandra/service/accord/AccordJournal.java | 8 1 file changed, 4 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 01/01: Address Benedict's comments
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19869
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit f39fecd6d9d33844b9cb19db82690517a54184be
Author: Alex Petrov
AuthorDate: Tue Aug 27 12:17:38 2024 +0200
Address Benedict's comments
---
accord-core/src/main/java/accord/local/PreLoadContext.java | 8
accord-core/src/main/java/accord/local/Status.java | 4 ++--
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/accord-core/src/main/java/accord/local/PreLoadContext.java
b/accord-core/src/main/java/accord/local/PreLoadContext.java
index e12b19d7..a9cf6d40 100644
--- a/accord-core/src/main/java/accord/local/PreLoadContext.java
+++ b/accord-core/src/main/java/accord/local/PreLoadContext.java
@@ -141,17 +141,17 @@ public interface PreLoadContext
static PreLoadContext contextFor(TxnId primary, Collection
additional, Seekables keys, KeyHistory keyHistory)
{
Invariants.checkState(!additional.contains(primary));
-return new Impl(primary, additional, keys, keyHistory);
+return new SimplePreLoadContext(primary, additional, keys, keyHistory);
}
-static class Impl implements PreLoadContext
+class SimplePreLoadContext implements PreLoadContext
{
private final TxnId primary;
private final Collection additional;
private final Seekables keys;
private final KeyHistory keyHistory;
-public Impl(TxnId primary, Collection additional, Seekables keys, KeyHistory keyHistory)
+public SimplePreLoadContext(TxnId primary, Collection
additional, Seekables keys, KeyHistory keyHistory)
{
this.primary = primary;
this.additional = additional;
@@ -161,7 +161,7 @@ public interface PreLoadContext
public String toString()
{
-return "Impl{" +
+return "SimplePreLoadContext{" +
"primary=" + primary +
", additional=" + additional +
", keys=" + keys +
diff --git a/accord-core/src/main/java/accord/local/Status.java
b/accord-core/src/main/java/accord/local/Status.java
index 5378a229..ccb5add4 100644
--- a/accord-core/src/main/java/accord/local/Status.java
+++ b/accord-core/src/main/java/accord/local/Status.java
@@ -882,7 +882,7 @@ public enum Status
public static Durability merge(Durability a, Durability b)
{
-int c = a == null ? -1 : a.compareTo(b);
+int c = a.compareTo(b);
if (c < 0) { Durability tmp = a; a = b; b = tmp; }
// if we know we are applied, we can remove the OrInvalidated
qualifier
if (a == UniversalOrInvalidated && (b == Majority || b ==
ShardUniversal || b == Local)) a = Universal;
@@ -894,7 +894,7 @@ public enum Status
public static Durability mergeAtLeast(Durability a, Durability b)
{
-int c = a == null ? -1 : a.compareTo(b);
+int c = a.compareTo(b);
if (c < 0) { Durability tmp = a; a = b; b = tmp; }
if (a == UniversalOrInvalidated && (b == Majority || b ==
ShardUniversal || b == Local)) a = Universal;
return a;
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch CASSANDRA-19869 created (now f39fecd6)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch CASSANDRA-19869 in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git at f39fecd6 Address Benedict's comments This branch includes the following new commits: new f39fecd6 Address Benedict's comments The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 01/03: Add an ability to reconstruct arbitrary epoch state from the log to TCM
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit b98954f0acb281101d42f76bad18a2051d114692
Author: Alex Petrov
AuthorDate: Fri Jul 19 18:01:16 2024 +0200
Add an ability to reconstruct arbitrary epoch state from the log to TCM
Patch by Alex Petrov; reviewed by Marcus Eriksson for CASSANDRA-19790.
---
.../org/apache/cassandra/metrics/TCMMetrics.java | 2 +
src/java/org/apache/cassandra/net/Verb.java| 3 +
.../schema/DistributedMetadataLogKeyspace.java | 35
.../cassandra/tcm/AbstractLocalProcessor.java | 2 +-
.../cassandra/tcm/AtomicLongBackedProcessor.java | 10 +++
.../cassandra/tcm/ClusterMetadataService.java | 5 ++
.../apache/cassandra/tcm/PaxosBackedProcessor.java | 5 ++
src/java/org/apache/cassandra/tcm/Processor.java | 4 +
.../apache/cassandra/tcm/ReconstructLogState.java | 86
.../org/apache/cassandra/tcm/RemoteProcessor.java | 24 ++
.../cassandra/tcm/StubClusterMetadataService.java | 6 ++
.../org/apache/cassandra/tcm/log/LocalLog.java | 7 +-
.../org/apache/cassandra/tcm/log/LogReader.java| 49 +++
.../org/apache/cassandra/tcm/log/LogStorage.java | 12 +++
.../cassandra/tcm/migration/GossipProcessor.java | 6 ++
.../test/log/CoordinatorPathTestBase.java | 5 ++
.../distributed/test/log/ReconstructEpochTest.java | 94 ++
.../distributed/test/log/TestProcessor.java| 6 ++
.../org/apache/cassandra/tcm/log/LocalLogTest.java | 5 ++
19 files changed, 364 insertions(+), 2 deletions(-)
diff --git a/src/java/org/apache/cassandra/metrics/TCMMetrics.java
b/src/java/org/apache/cassandra/metrics/TCMMetrics.java
index 134a1a34e2..68d5db55fe 100644
--- a/src/java/org/apache/cassandra/metrics/TCMMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/TCMMetrics.java
@@ -62,6 +62,7 @@ public class TCMMetrics
public final Meter progressBarrierCLRelax;
public final Meter coordinatorBehindSchema;
public final Meter coordinatorBehindPlacements;
+public final Meter reconstructLogStateCall;
private TCMMetrics()
{
@@ -113,6 +114,7 @@ public class TCMMetrics
coordinatorBehindSchema =
Metrics.meter(factory.createMetricName("CoordinatorBehindSchema"));
coordinatorBehindPlacements =
Metrics.meter(factory.createMetricName("CoordinatorBehindPlacements"));
+reconstructLogStateCall =
Metrics.meter(factory.createMetricName("ReconstructLogStateCall"));
}
public void recordCommitFailureLatency(long latency, TimeUnit timeUnit,
boolean isRejection)
diff --git a/src/java/org/apache/cassandra/net/Verb.java
b/src/java/org/apache/cassandra/net/Verb.java
index 151b59c8df..c4d87139ea 100644
--- a/src/java/org/apache/cassandra/net/Verb.java
+++ b/src/java/org/apache/cassandra/net/Verb.java
@@ -130,6 +130,7 @@ import org.apache.cassandra.tcm.Discovery;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.FetchCMSLog;
import org.apache.cassandra.tcm.FetchPeerLog;
+import org.apache.cassandra.tcm.ReconstructLogState;
import org.apache.cassandra.tcm.migration.Election;
import org.apache.cassandra.tcm.sequences.DataMovements;
import org.apache.cassandra.tcm.serialization.MessageSerializers;
@@ -298,6 +299,8 @@ public enum Verb
TCM_DISCOVER_REQ (813, P0, rpcTimeout, INTERNAL_METADATA,()
-> NoPayload.serializer, () ->
Discovery.instance.requestHandler,TCM_DISCOVER_RSP ),
TCM_FETCH_PEER_LOG_RSP (818, P0, rpcTimeout, FETCH_LOG,
MessageSerializers::logStateSerializer, RESPONSE_HANDLER
),
TCM_FETCH_PEER_LOG_REQ (819, P0, rpcTimeout, FETCH_LOG,()
-> FetchPeerLog.serializer, () ->
FetchPeerLog.Handler.instance,TCM_FETCH_PEER_LOG_RSP ),
+TCM_RECONSTRUCT_EPOCH_RSP (820, P0, rpcTimeout, FETCH_LOG,
MessageSerializers::logStateSerializer, () ->
ResponseVerbHandler.instance ),
+TCM_RECONSTRUCT_EPOCH_REQ (821, P0, rpcTimeout, FETCH_LOG,()
-> ReconstructLogState.serializer, () ->
ReconstructLogState.Handler.instance, TCM_FETCH_PEER_LOG_RSP ),
INITIATE_DATA_MOVEMENTS_RSP (814, P1, rpcTimeout, MISC, () ->
NoPayload.serializer, RESPONSE_HANDLER
),
INITIATE_DATA_MOVEMENTS_REQ (815, P1, rpcTimeout, MISC, () ->
DataMovement.serializer, () -> DataMovementVerbHandler.instance,
INITIATE_DATA_MOVEMENTS_RSP ),
diff --git
a/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java
b/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java
index 26852
(cassandra) 02/03: Switch to infinite loop executor instead of a while-loop thread.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 6b2bd52447f9dd97e890069cb3aa5eb2e53a400f
Author: Alex Petrov
AuthorDate: Mon Aug 26 12:51:29 2024 +0200
Switch to infinite loop executor instead of a while-loop thread.
Patch by Alex Petrov; reviewed by David Capwell for CASSANDRA-19864
---
.../cassandra/service/accord/AccordJournal.java| 142 -
1 file changed, 83 insertions(+), 59 deletions(-)
diff --git a/src/java/org/apache/cassandra/service/accord/AccordJournal.java
b/src/java/org/apache/cassandra/service/accord/AccordJournal.java
index 956473cfd8..3ca7277d39 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordJournal.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordJournal.java
@@ -52,6 +52,8 @@ import accord.primitives.TxnId;
import accord.utils.Invariants;
import org.agrona.collections.Long2ObjectHashMap;
import org.agrona.collections.LongArrayList;
+import org.apache.cassandra.concurrent.InfiniteLoopExecutor;
+import org.apache.cassandra.concurrent.Interruptible;
import org.apache.cassandra.concurrent.ManyToOneConcurrentLinkedQueue;
import org.apache.cassandra.concurrent.Shutdownable;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -105,6 +107,9 @@ import static
accord.messages.MessageType.SET_SHARD_DURABLE_REQ;
import static accord.messages.MessageType.STABLE_FAST_PATH_REQ;
import static accord.messages.MessageType.STABLE_MAXIMAL_REQ;
import static accord.messages.MessageType.STABLE_SLOW_PATH_REQ;
+import static
org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static
org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+import static org.apache.cassandra.concurrent.Interruptible.State.NORMAL;
import static
org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType.INTEROP_APPLY_MAXIMAL_REQ;
import static
org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType.INTEROP_APPLY_MINIMAL_REQ;
import static
org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType.INTEROP_COMMIT_MAXIMAL_REQ;
@@ -165,7 +170,7 @@ public class AccordJournal implements IJournal, Shutdownable
{
Invariants.checkState(status == Status.STARTED);
status = Status.TERMINATING;
-delayedRequestProcessor.runOnce();
+delayedRequestProcessor.shutdown();
journal.shutdown();
status = Status.TERMINATED;
}
@@ -625,12 +630,18 @@ public class AccordJournal implements IJournal,
Shutdownable
* Handling topology changes / epoch shift
*/
-private final class DelayedRequestProcessor extends Thread
+private class DelayedRequestProcessor implements Interruptible.Task
{
private final ManyToOneConcurrentLinkedQueue
delayedRequests = new ManyToOneConcurrentLinkedQueue<>();
private final LongArrayList waitForEpochs = new LongArrayList();
private final Long2ObjectHashMap> byEpoch = new
Long2ObjectHashMap<>();
private final AtomicReference signal = new
AtomicReference<>(Condition.newOneTimeCondition());
+private Interruptible executor;
+
+public void start()
+{
+ executor =
executorFactory().infiniteLoop("AccordJournal-delayed-request-processor",
this::run, SAFE, InfiniteLoopExecutor.Daemon.NON_DAEMON,
InfiniteLoopExecutor.Interrupts.SYNCHRONIZED);
+}
private void delay(RequestContext requestContext)
{
@@ -643,81 +654,94 @@ public class AccordJournal implements IJournal,
Shutdownable
signal.get().signal();
}
-public void run()
+@Override
+public void run(Interruptible.State state) throws InterruptedException
{
-while (!Thread.currentThread().isInterrupted() &&
isRunnable(status))
+if (state != NORMAL || !Thread.currentThread().isInterrupted() &&
!isRunnable(status))
+return;
+
+try
{
-try
+Condition signal = Condition.newOneTimeCondition();
+this.signal.set(signal);
+// First, poll delayed requests, put them into by epoch
+while (!delayedRequests.isEmpty())
{
-Condition signal = Condition.newOneTimeCondition();
-this.signal.set(signal);
-// First, poll delayed requests, put them into by epoch
-while (!delayedRequests.isEmpty())
+RequestContext context = delayedRequests.poll();
+long waitForEpoch = context.waitForEpoch;
+
+List l =
byEpoch.computeIfAbsent(waitForEpoch, (ignore) -> new ArrayList<>());
+if (l.is
(cassandra) 03/03: Revert acccord module to absolute path
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git commit bfadfdc123dcb5d90a1a3a34c970056d5c57eb21 Author: Alex Petrov AuthorDate: Mon Aug 26 12:54:54 2024 +0200 Revert acccord module to absolute path --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 60a9510e7a..616dacf610 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "modules/accord"] path = modules/accord - url = ../cassandra-accord.git + url = https://github.com/apache/cassandra-accord.git branch = trunk - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cep-15-accord updated (8c8d271bb9 -> bfadfdc123)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git from 8c8d271bb9 CommandsForRanges does not support slice which cause over returned data being sent new b98954f0ac Add an ability to reconstruct arbitrary epoch state from the log to TCM new 6b2bd52447 Switch to infinite loop executor instead of a while-loop thread. new bfadfdc123 Revert acccord module to absolute path The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .gitmodules| 2 +- .../org/apache/cassandra/metrics/TCMMetrics.java | 2 + src/java/org/apache/cassandra/net/Verb.java| 3 + .../schema/DistributedMetadataLogKeyspace.java | 35 + .../cassandra/service/accord/AccordJournal.java| 142 - .../cassandra/tcm/AbstractLocalProcessor.java | 2 +- .../cassandra/tcm/AtomicLongBackedProcessor.java | 10 ++ .../cassandra/tcm/ClusterMetadataService.java | 5 + .../apache/cassandra/tcm/PaxosBackedProcessor.java | 5 + src/java/org/apache/cassandra/tcm/Processor.java | 4 + .../apache/cassandra/tcm/ReconstructLogState.java | 86 + .../org/apache/cassandra/tcm/RemoteProcessor.java | 24 .../cassandra/tcm/StubClusterMetadataService.java | 6 + .../org/apache/cassandra/tcm/log/LocalLog.java | 7 +- .../org/apache/cassandra/tcm/log/LogReader.java| 49 +++ .../org/apache/cassandra/tcm/log/LogStorage.java | 12 ++ .../cassandra/tcm/migration/GossipProcessor.java | 6 + .../test/log/CoordinatorPathTestBase.java | 5 + .../distributed/test/log/ReconstructEpochTest.java | 94 ++ .../distributed/test/log/TestProcessor.java| 6 + .../org/apache/cassandra/tcm/log/LocalLogTest.java | 5 + 21 files changed, 448 insertions(+), 62 deletions(-) create mode 100644 src/java/org/apache/cassandra/tcm/ReconstructLogState.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/log/ReconstructEpochTest.java - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch CASSANDRA-19865 created (now 61088d1e)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch CASSANDRA-19865 in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git at 61088d1e Fix CompactionIteratorTest This branch includes the following new commits: new 61088d1e Fix CompactionIteratorTest The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) 01/01: Fix CompactionIteratorTest
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-19865
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 61088d1ec1d3fb5252a31e21cd0ccc85c29936ba
Author: Alex Petrov
AuthorDate: Wed Aug 14 12:41:34 2024 +0200
Fix CompactionIteratorTest
---
accord-core/src/main/java/accord/primitives/TxnId.java | 8
1 file changed, 8 insertions(+)
diff --git a/accord-core/src/main/java/accord/primitives/TxnId.java
b/accord-core/src/main/java/accord/primitives/TxnId.java
index 63824421..8c3e9c5f 100644
--- a/accord-core/src/main/java/accord/primitives/TxnId.java
+++ b/accord-core/src/main/java/accord/primitives/TxnId.java
@@ -21,6 +21,8 @@ package accord.primitives;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.annotations.VisibleForTesting;
+
import accord.local.Node.Id;
import accord.primitives.Routable.Domain;
import accord.primitives.Txn.Kind;
@@ -104,6 +106,12 @@ public class TxnId extends Timestamp
return new TxnId(epoch(), hlc(), kind, domain(), node);
}
+@VisibleForTesting
+public TxnId as(Kind kind, Domain domain)
+{
+return new TxnId(epoch(), hlc(), kind, domain, node);
+}
+
public TxnId withEpoch(long epoch)
{
return epoch == epoch() ? this : new TxnId(epoch, hlc(), flags(),
node);
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch trunk updated: Fix ProgressBarrierTest
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new c263cdf610 Fix ProgressBarrierTest
c263cdf610 is described below
commit c263cdf6106adbf706bb85991e103e2c6519946c
Author: Alex Petrov
AuthorDate: Tue Aug 20 16:39:16 2024 +0200
Fix ProgressBarrierTest
Patch by Alex Petrov; reviewed by Marcus Eriksson for CASSANDRA-19849.
---
.../org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git
a/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java
b/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java
index 92610ce6d3..c5bf2c129b 100644
--- a/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java
+++ b/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java
@@ -114,7 +114,9 @@ public class ProgressBarrierTest extends CMSTestBase
{
List allNodes = new ArrayList<>();
TokenPlacementModel.Node node = null;
-int nodesInCluster = Math.max(rf.total(), nodes.get());
+// + 1 since one of the nodes will not be joined yet by the
time we create progress barrier, which will fail
+// a check with ALL.
+int nodesInCluster = Math.max(rf.total(), nodes.get()) + 1;
for (int i = 1; i <= nodesInCluster; i++)
{
node = nodeFactory.make(i, (i % rf.dcs()) + 1, 1);
@@ -176,7 +178,7 @@ public class ProgressBarrierTest extends CMSTestBase
Assert.assertTrue(String.format("Should have
collected at least %d nodes but got %d." +
"\nRF: %s" +
"\nReplicas: %s" +
-"\nNodes: %s",
expected, collected.size(), rf, replicas, collected),
+"\nNodes:%s",
expected, collected.size(), rf, replicas, collected),
collected.size() >= expected);
break;
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch trunk updated: Simplify and deduplicate Harry ModelChecker
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 52396ec1fc Simplify and deduplicate Harry ModelChecker
52396ec1fc is described below
commit 52396ec1fc4d08a988714704ff0cbb796ee06f7f
Author: Alex Petrov
AuthorDate: Fri Jul 19 14:44:31 2024 +0200
Simplify and deduplicate Harry ModelChecker
Patch by Alex Petrov; reviewed by Caleb Rackliffe for CASSANDRA-19788.
---
.../cassandra/concurrent/InfiniteLoopExecutor.java | 13 +-
.../test/log/MetadataChangeSimulationTest.java | 10 +-
.../distributed/test/log/ModelChecker.java | 307 -
.../test/log/PlacementSimulatorTest.java | 7 +-
.../cassandra/distributed/test/log/RngUtils.java | 106 ---
.../harry/examples/RangeTombstoneBurnTest.java | 55 ++--
.../fuzz/harry/examples/RepairBurnTest.java| 11 +-
.../fuzz/harry/integration/ddl/SchemaGenTest.java | 5 +-
.../dsl/HistoryBuilderIntegrationTest.java | 92 +++---
.../HistoryBuilderOverridesIntegrationTest.java| 8 +-
.../model/QuiescentCheckerIntegrationTest.java | 24 +-
.../model/ReconcilerIntegrationTest.java | 2 +-
.../model/reconciler/SimpleReconcilerTest.java | 2 +-
.../cassandra/fuzz/sai/SingleNodeSAITest.java | 2 +-
.../cassandra/harry/checker/ModelChecker.java | 287 +++
.../apache/cassandra/harry/dsl/HistoryBuilder.java | 18 +-
.../harry/dsl/SingleOperationVisitBuilder.java | 2 +-
.../cassandra/harry/model/QuiescentChecker.java| 36 ++-
.../model/QuiescentLocalStateCheckerBase.java | 2 +-
.../apache/cassandra/harry/model/SelectHelper.java | 14 +-
.../harry/model/reconciler/PartitionState.java | 2 +-
.../harry/model/reconciler/Reconciler.java | 31 ++-
.../apache/cassandra/harry/operations/Query.java | 80 +-
.../cassandra/harry/operations/QueryGenerator.java | 2 +-
.../harry/visitors/AllPartitionsValidator.java | 14 +-
.../harry/visitors/CorruptingVisitor.java | 2 +-
.../cassandra/harry/visitors/SingleValidator.java | 2 +-
.../simulator/test/HarrySimulatorTest.java | 2 +-
.../tcm/sequences/ProgressBarrierTest.java | 4 +-
29 files changed, 496 insertions(+), 646 deletions(-)
diff --git a/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java
b/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java
index 51c5f9f69e..ac10a70c30 100644
--- a/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java
+++ b/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java
@@ -149,10 +149,21 @@ public class InfiniteLoopExecutor implements Interruptible
interruptHandler.accept(thread);
}
+public void shutdownGracefully()
+{
+stateUpdater.updateAndGet(this, cur -> cur != TERMINATED && cur !=
SHUTTING_DOWN_NOW ? SHUTTING_DOWN : cur);
+}
+
public void shutdown()
+{
+shutdown(true);
+}
+
+public void shutdown(boolean interrupt)
{
stateUpdater.updateAndGet(this, cur -> cur != TERMINATED && cur !=
SHUTTING_DOWN_NOW ? SHUTTING_DOWN : cur);
-interruptHandler.accept(thread);
+if (interrupt)
+interruptHandler.accept(thread);
}
public Object shutdownNow()
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
index 26d1cff7ac..d5c25fe6d2 100644
---
a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
+++
b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
@@ -35,7 +35,6 @@ import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.google.common.collect.Sets;
-import org.apache.cassandra.harry.sut.TokenPlacementModel.DCReplicas;
import org.junit.Assert;
import org.junit.Test;
@@ -48,7 +47,10 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.harry.checker.ModelChecker;
+import org.apache.cassandra.harry.gen.EntropySource;
import org.apache.cassandra.harry.sut.TokenPlacementModel;
+import org.apache.cassandra.harry.sut.TokenPlacementModel.DCReplicas;
import org.apache.cassandra.locator.CMSPlacementStrategy;
import org.apache.cassandra.locator.EndpointsForRange;
import org.apache.cassandra.locator.InetAddressAndPort;
@@ -646,17 +648,17 @@ public class MetadataChangeSimulationTest extends
CMSTestBase
return pair(newState, node);
}
-private No
(cassandra) branch trunk updated (53c6a106ac -> 0e63080d3a)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git from 53c6a106ac Merge branch 'cassandra-5.0' into trunk add ff3e5ab76d Ninja: Fix test merge after fixing CASSANDRA-19534 add 60b5a3c815 Merge branch 'cassandra-4.1' into cassandra-5.0 new 0e63080d3a Merge branch 'cassandra-5.0' into trunk The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cassandra-4.1 updated (aa7afeabce -> ff3e5ab76d)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cassandra-4.1 in repository https://gitbox.apache.org/repos/asf/cassandra.git from aa7afeabce Merge branch 'cassandra-4.0' into cassandra-4.1 add ff3e5ab76d Ninja: Fix test merge after fixing CASSANDRA-19534 No new revisions were added by this update. Summary of changes: .../org/apache/cassandra/distributed/test/ReadRepairTest.java| 1 + 1 file changed, 1 insertion(+) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cassandra-5.0 updated (c137958963 -> 60b5a3c815)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cassandra-5.0 in repository https://gitbox.apache.org/repos/asf/cassandra.git from c137958963 Minor improvements in Cassandra shutdown and startup logs add ff3e5ab76d Ninja: Fix test merge after fixing CASSANDRA-19534 add 60b5a3c815 Merge branch 'cassandra-4.1' into cassandra-5.0 No new revisions were added by this update. Summary of changes: - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 01/01: Merge branch 'cassandra-5.0' into trunk
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git commit 0e63080d3a44c28e86f75e183ceb2c91bdb45504 Merge: 53c6a106ac 60b5a3c815 Author: Alex Petrov AuthorDate: Wed Aug 14 09:45:40 2024 +0200 Merge branch 'cassandra-5.0' into trunk - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch trunk updated: NPE on Directory in ShortPaxosSimulationTest
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 018feb36f4 NPE on Directory in ShortPaxosSimulationTest
018feb36f4 is described below
commit 018feb36f42d28bc11073741ef224b42f9a60ad0
Author: Alex Petrov
AuthorDate: Thu Jul 25 14:47:13 2024 +0200
NPE on Directory in ShortPaxosSimulationTest
Patch by Alex Petrov, reviewed by Caleb Rackliffe and Marcus Eriksson for
CASSANDRA-19794
---
src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java| 6 +++---
.../tcm/transformations/cms/PrepareCMSReconfiguration.java | 3 +++
.../org/apache/cassandra/simulator/cluster/OnClusterLeave.java | 7 ++-
.../org/apache/cassandra/simulator/cluster/OnClusterReplace.java | 7 ++-
4 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java
b/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java
index 6a90aa1770..400553ce01 100644
--- a/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java
+++ b/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java
@@ -106,13 +106,13 @@ public interface CMSPlacementStrategy
rf.put(e.getKey(), ReplicationFactor.fullOnly(e.getValue()));
}
-Directory directory = metadata.directory;
+Directory tmpDirectory = metadata.directory;
TokenMap tokenMap = metadata.tokenMap;
for (NodeId peerId : metadata.directory.peerIds())
{
if (!filter.apply(metadata, peerId))
{
-directory = directory.without(peerId);
+tmpDirectory = tmpDirectory.without(peerId);
tokenMap = tokenMap.unassignTokens(peerId);
}
}
@@ -123,7 +123,7 @@ public interface CMSPlacementStrategy
Token minToken =
DatabaseDescriptor.getPartitioner().getMinimumToken();
EndpointsForRange endpoints =
NetworkTopologyStrategy.calculateNaturalReplicas(minToken,
new Range<>(minToken, minToken),
-
directory,
+
tmpDirectory,
tokenMap,
rf);
diff --git
a/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java
b/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java
index 7daee231b1..0367640fda 100644
---
a/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java
+++
b/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java
@@ -215,6 +215,9 @@ public class PrepareCMSReconfiguration
public static Diff diff(Set currentCms, Set newCms)
{
+assert !currentCms.contains(null) : "Current CMS contains a null value
" + currentCms;
+assert !newCms.contains(null) : "New CMS contains a null value " +
newCms;
+
List additions = new ArrayList<>();
for (NodeId node : newCms)
{
diff --git
a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterLeave.java
b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterLeave.java
index 9e77b55815..2b2ce15086 100644
---
a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterLeave.java
+++
b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterLeave.java
@@ -30,9 +30,11 @@ import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.MultiStepOperation;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.sequences.LeaveStreams;
+import org.apache.cassandra.tcm.sequences.ReconfigureCMS;
import org.apache.cassandra.tcm.sequences.UnbootstrapAndLeave;
import org.apache.cassandra.tcm.transformations.PrepareLeave;
+import static
org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
import static org.apache.cassandra.utils.LazyToString.lazy;
class OnClusterLeave extends OnClusterChangeTopology
@@ -83,6 +85,9 @@ class OnClusterLeave extends OnClusterChangeTopology
{
super("Prepare Leave", actions, on, () -> {
ClusterMetadata metadata = ClusterMetadata.current();
+ReconfigureCMS.maybeReconfigureCMS(metadata,
getBroadcastAddressAndPort());
+
+
(cassandra-in-jvm-dtest-api) branch trunk updated (b9f2d0a -> 71f854c)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra-in-jvm-dtest-api.git from b9f2d0a Ninja: Fix project version in pom.xml new d7b9c68 CASSANDRA-19783 InstanceClassLoader leak detection new 71f854c Fix formatting The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../distributed/shared/InstanceClassLoader.java| 19 + .../shared/InstanceClassLoaderTest.java| 81 ++ 2 files changed, 100 insertions(+) create mode 100644 src/test/java/org/apache/cassandra/distributed/shared/InstanceClassLoaderTest.java - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-in-jvm-dtest-api) 01/02: CASSANDRA-19783 InstanceClassLoader leak detection
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-in-jvm-dtest-api.git
commit d7b9c68705b380b45068d5461b389f7244b9a6aa
Author: Doug Rohrer
AuthorDate: Thu Jul 18 13:18:32 2024 -0400
CASSANDRA-19783 InstanceClassLoader leak detection
This commit adds a WeakHashMap-backed set of InstanceClassLoaders which can
be used to
count the live instances of classloaders. This, in turn, is used in
ResourceLeakTest
in Cassandra's main source to detect leaking InstanceClassLoaders, and will
allow us
to find them in CI before a commit gets through that adds a new leak.
---
.../distributed/shared/InstanceClassLoader.java| 17 +
.../shared/InstanceClassLoaderTest.java| 77 ++
2 files changed, 94 insertions(+)
diff --git
a/src/main/java/org/apache/cassandra/distributed/shared/InstanceClassLoader.java
b/src/main/java/org/apache/cassandra/distributed/shared/InstanceClassLoader.java
index a2381f1..9443442 100644
---
a/src/main/java/org/apache/cassandra/distributed/shared/InstanceClassLoader.java
+++
b/src/main/java/org/apache/cassandra/distributed/shared/InstanceClassLoader.java
@@ -31,8 +31,11 @@ import java.net.URLConnection;
import java.security.CodeSigner;
import java.security.CodeSource;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.WeakHashMap;
import java.util.function.Predicate;
import java.util.jar.Manifest;
@@ -56,6 +59,12 @@ public class InstanceClassLoader extends URLClassLoader
|| name.startsWith("org.jboss.byteman.")
|| name.startsWith("oshi.jna.");
+// Use a WeakHashMap to get the approximate count of live classloaders.
+// Once the classloader is otherwise unused, GC will (eventually) remove
it from this set as well.
+// NOTE: Because potentially incompletely-initialized loaders are
published to this set, do not expose the set
+// itself other than through the getApproximateLiveLoaderCount method.
+private static final Set liveLoaders =
Collections.synchronizedSet(
+
Collections.newSetFromMap(new WeakHashMap<>()));
private volatile boolean isClosed = false;
private final URL[] urls;
private final int generation; // used to help debug class loader leaks, by
helping determine which classloaders should have been collected
@@ -88,6 +97,7 @@ public class InstanceClassLoader extends URLClassLoader
this.id = id;
this.loadShared = loadShared == null ? DEFAULT_SHARED_PACKAGES :
loadShared;
this.transform = transform;
+liveLoaders.add(this);
}
public static Predicate getDefaultLoadSharedFilter()
@@ -95,6 +105,13 @@ public class InstanceClassLoader extends URLClassLoader
return DEFAULT_SHARED_PACKAGES;
}
+public static int getApproximateLiveLoaderCount(boolean forceGC) {
+if (forceGC) {
+System.gc();
+}
+return liveLoaders.size();
+}
+
public int getClusterGeneration()
{
return generation;
diff --git
a/src/test/java/org/apache/cassandra/distributed/shared/InstanceClassLoaderTest.java
b/src/test/java/org/apache/cassandra/distributed/shared/InstanceClassLoaderTest.java
new file mode 100644
index 000..da58bdc
--- /dev/null
+++
b/src/test/java/org/apache/cassandra/distributed/shared/InstanceClassLoaderTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.shared;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class InstanceClassLoaderTest
+{
+
+private static final int NUM_LOADERS = 10;
+private List loaders = new ArrayList<>();
+
+@BeforeEach
+public
(cassandra-in-jvm-dtest-api) 02/02: Fix formatting
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-in-jvm-dtest-api.git
commit 71f854cfe0050da9dc7ffd9e52d0ef357decbd39
Author: Doug Rohrer
AuthorDate: Thu Jul 18 16:16:35 2024 -0400
Fix formatting
---
.../distributed/shared/InstanceClassLoader.java| 6 ++-
.../shared/InstanceClassLoaderTest.java| 50 --
2 files changed, 31 insertions(+), 25 deletions(-)
diff --git
a/src/main/java/org/apache/cassandra/distributed/shared/InstanceClassLoader.java
b/src/main/java/org/apache/cassandra/distributed/shared/InstanceClassLoader.java
index 9443442..6bb43cc 100644
---
a/src/main/java/org/apache/cassandra/distributed/shared/InstanceClassLoader.java
+++
b/src/main/java/org/apache/cassandra/distributed/shared/InstanceClassLoader.java
@@ -105,8 +105,10 @@ public class InstanceClassLoader extends URLClassLoader
return DEFAULT_SHARED_PACKAGES;
}
-public static int getApproximateLiveLoaderCount(boolean forceGC) {
-if (forceGC) {
+public static int getApproximateLiveLoaderCount(boolean forceGC)
+{
+if (forceGC)
+{
System.gc();
}
return liveLoaders.size();
diff --git
a/src/test/java/org/apache/cassandra/distributed/shared/InstanceClassLoaderTest.java
b/src/test/java/org/apache/cassandra/distributed/shared/InstanceClassLoaderTest.java
index da58bdc..e6fb61a 100644
---
a/src/test/java/org/apache/cassandra/distributed/shared/InstanceClassLoaderTest.java
+++
b/src/test/java/org/apache/cassandra/distributed/shared/InstanceClassLoaderTest.java
@@ -32,46 +32,50 @@ public class InstanceClassLoaderTest
{
private static final int NUM_LOADERS = 10;
-private List loaders = new ArrayList<>();
+private final List loaders = new ArrayList<>();
+
+@Test
+public void testRefCountZeroWhenNoneCreated()
+{
+
assertThat(InstanceClassLoader.getApproximateLiveLoaderCount(true)).isEqualTo(0);
+}
+
+@Test
+public void testRefCountWhenInstancesCreatedAndStillOnHeap()
+{
+createClassLoadersAndAssertCount();
+}
+
+@Test
+public void testRefCountAfterInstancesCanBeGced()
+{
+createClassLoadersAndAssertCount();
+loaders.clear();
+assertThat(waitForZeroClassLoaders(10)).isTrue();
+}
@BeforeEach
-public void ensureNoClassLoadersOnStart() {
+public void ensureNoClassLoadersOnStart()
+{
assertThat(waitForZeroClassLoaders(50)).isTrue();
}
public boolean waitForZeroClassLoaders(int gcAttempts)
{
int i = 0;
-while (InstanceClassLoader.getApproximateLiveLoaderCount(true) > 0 &&
i++ < gcAttempts) {
+while (InstanceClassLoader.getApproximateLiveLoaderCount(true) > 0 &&
i++ < gcAttempts)
+{
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
}
return InstanceClassLoader.getApproximateLiveLoaderCount(false) == 0;
-
-}
-
-@Test
-public void testRefCountZeroWhenNoneCreated() {
-
assertThat(InstanceClassLoader.getApproximateLiveLoaderCount(true)).isEqualTo(0);
-}
-
-@Test
-public void testRefCountWhenInstancesCreatedAndStillOnHeap()
-{
-createClassLoadersAndAssertCount();
}
private void createClassLoadersAndAssertCount()
{
-for (int i = 0; i < NUM_LOADERS; i++) {
+for (int i = 0; i < NUM_LOADERS; i++)
+{
loaders.add(new InstanceClassLoader(0, 0, new URL[0],
this.getClass().getClassLoader()));
}
assertThat(InstanceClassLoader.getApproximateLiveLoaderCount(true)).isEqualTo(NUM_LOADERS);
}
-
-@Test
-public void testRefCountAfterInstancesCanBeGced() {
-createClassLoadersAndAssertCount();
-loaders.clear();
-assertThat(waitForZeroClassLoaders(10)).isTrue();
-}
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-accord) branch trunk updated: Accord Journal / Determinism
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4c870dc9 Accord Journal / Determinism
4c870dc9 is described below
commit 4c870dc9b561a841ea9b923ff739953adcc00325
Author: Alex Petrov
AuthorDate: Mon Jun 17 09:53:43 2024 +0200
Accord Journal / Determinism
* Store intermediate Command states in the log
* Load Command states from the log
Patch by Alex Petrov; reviewed by Benedict Elliott Smith for CASSANDRA-19757
---
.../java/accord/impl/InMemoryCommandStore.java | 13 -
.../src/main/java/accord/local/Cleanup.java| 1 -
.../src/main/java/accord/local/Command.java| 18 +-
.../src/main/java/accord/local/Listeners.java | 2 +-
.../main/java/accord/local/SafeCommandStore.java | 5 +
.../main/java/accord/local/SerializerSupport.java | 613 -
.../src/main/java/accord/utils/Invariants.java | 7 +
.../accord/impl/basic/DelayedCommandStores.java| 49 +-
.../src/test/java/accord/impl/basic/Journal.java | 560 +++
.../java/accord/local/BootstrapLocalTxnTest.java | 3 +-
.../test/java/accord/local/CheckedCommands.java| 35 ++
.../java/accord/utils/ReducingRangeMapTest.java| 4 -
12 files changed, 401 insertions(+), 909 deletions(-)
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index 5ee3f8d5..4932b181 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -461,7 +461,6 @@ public abstract class InMemoryCommandStore extends
CommandStore
private T executeInContext(InMemoryCommandStore commandStore,
PreLoadContext preLoadContext, Function function,
boolean isDirectCall)
{
-
SafeCommandStore safeStore =
commandStore.beginOperation(preLoadContext);
try
{
@@ -623,18 +622,6 @@ public abstract class InMemoryCommandStore extends
CommandStore
}
}
-private static class TimestampAndStatus
-{
-public final Timestamp timestamp;
-public final Status status;
-
-public TimestampAndStatus(Timestamp timestamp, Status status)
-{
-this.timestamp = timestamp;
-this.status = status;
-}
-}
-
public static class GlobalTimestampsForKey extends
GlobalState
{
private final Key key;
diff --git a/accord-core/src/main/java/accord/local/Cleanup.java
b/accord-core/src/main/java/accord/local/Cleanup.java
index bb6b1d2e..5da56e3c 100644
--- a/accord-core/src/main/java/accord/local/Cleanup.java
+++ b/accord-core/src/main/java/accord/local/Cleanup.java
@@ -34,7 +34,6 @@ import static
accord.local.SaveStatus.TruncatedApplyWithOutcome;
import static accord.local.SaveStatus.Uninitialised;
import static accord.local.Status.Applied;
import static accord.local.Status.Durability.Majority;
-import static accord.local.Status.Durability.Universal;
import static accord.local.Status.Durability.UniversalOrInvalidated;
import static accord.local.Status.PreCommitted;
import static accord.primitives.Txn.Kind.EphemeralRead;
diff --git a/accord-core/src/main/java/accord/local/Command.java
b/accord-core/src/main/java/accord/local/Command.java
index 96df4461..51bd9410 100644
--- a/accord-core/src/main/java/accord/local/Command.java
+++ b/accord-core/src/main/java/accord/local/Command.java
@@ -69,7 +69,7 @@ import static java.lang.String.format;
public abstract class Command implements CommonAttributes
{
-interface Listener
+public interface Listener
{
void onChange(SafeCommandStore safeStore, SafeCommand safeCommand);
@@ -275,6 +275,7 @@ public abstract class Command implements CommonAttributes
this.listeners = common.durableListeners();
}
+
@Override
public boolean equals(Object o)
{
@@ -645,9 +646,15 @@ public abstract class Command implements CommonAttributes
SaveStatus saveStatus = saveStatus();
return saveStatus.hasBeen(Status.Committed) &&
!saveStatus.hasBeen(Invalidated);
}
+
public final boolean isStable()
{
SaveStatus saveStatus = saveStatus();
+return isStable(saveStatus);
+}
+
+public static boolean isStable(SaveStatus saveStatus)
+{
return saveStatus.hasBeen(Status.Stable) &&
!saveStatus.hasBeen(Invalidated);
}
@@ -661,11 +668,6 @@ public abstract class Command implements CommonAttributes
return Invariants.cast(this, Executed.class);
}
-public final boolean isTruncated()
-{
-return status().hasBeen(Status.Truncated);
-}
-
public abstract Command
(cassandra) branch trunk updated: Simulator: Add instrumentation for Semaphore
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 93ddd3a976 Simulator: Add instrumentation for Semaphore
93ddd3a976 is described below
commit 93ddd3a976a3a1dc6694eba7330598f5f83d2fec
Author: Alex Petrov
AuthorDate: Tue Jul 23 17:58:12 2024 +0200
Simulator: Add instrumentation for Semaphore
Patch by Alex Petrov; reviewed by David Capwell for CASSANDRA-19695.
---
.../org/apache/cassandra/utils/MonotonicClock.java | 9 +-
.../distributed/test/log/CMSTestBase.java | 9 +-
.../systems/InterceptingGlobalMethods.java | 13 ++
.../simulator/systems/InterceptingSemaphore.java | 197 +
.../systems/InterceptorOfGlobalMethods.java| 27 ++-
.../cassandra/simulator/test/SemaphoreTest.java| 154
.../simulator/test/SimulationTestBase.java | 51 +-
.../simulator/test/TrivialSimulationTest.java | 14 +-
8 files changed, 458 insertions(+), 16 deletions(-)
diff --git a/src/java/org/apache/cassandra/utils/MonotonicClock.java
b/src/java/org/apache/cassandra/utils/MonotonicClock.java
index 7be54c008b..c68ed5d9ef 100644
--- a/src/java/org/apache/cassandra/utils/MonotonicClock.java
+++ b/src/java/org/apache/cassandra/utils/MonotonicClock.java
@@ -248,7 +248,14 @@ public interface MonotonicClock
public static class SystemClock extends AbstractEpochSamplingClock
{
-private SystemClock()
+// Without making this constructor public you may start getting the
following exception in the simulator:
+//java.lang.IncompatibleClassChangeError: Type
+// org.apache.cassandra.utils.MonotonicClock$Global is not a
nest member of
+// org.apache.cassandra.utils.MonotonicClock: types are in
different packages
+// There might be a problem with a simulator and how we allow access,
but I verified the change access
+// flags on method of the
org/apache/cassandra/utils/MonotonicClock$SystemClock
+// class to ACC_PUBLIC, and ensured proper testing relationship from
both the surrounding and nested class.
+public SystemClock()
{
super(Clock.Global::currentTimeMillis);
}
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java
b/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java
index c1975cdbfe..e254798915 100644
---
a/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java
+++
b/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java
@@ -91,10 +91,15 @@ public class CMSTestBase
public final TokenPlacementModel.ReplicationFactor rf;
public CMSSut(IIsolatedExecutor.SerializableFunction processorFactory, boolean addListeners,
TokenPlacementModel.ReplicationFactor rf)
+{
+this(processorFactory, addListeners,
Mockito.mock(SchemaProvider.class), rf);
+}
+
+public CMSSut(IIsolatedExecutor.SerializableFunction processorFactory, boolean addListeners, SchemaProvider
schemaProvider, TokenPlacementModel.ReplicationFactor rf)
{
partitioner = Murmur3Partitioner.instance;
this.rf = rf;
-schemaProvider = Mockito.mock(SchemaProvider.class);
+this.schemaProvider = schemaProvider;
ClusterMetadata initial = new ClusterMetadata(partitioner);
log = LocalLog.logSpec()
.sync()
@@ -127,7 +132,7 @@ public class CMSTestBase
}, schemaProvider));
}
-public void close() throws Exception
+public void close()
{
ClusterMetadataService.unsetInstance();
}
diff --git
a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java
b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java
index abc71eae35..34c0f6bacc 100644
---
a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java
+++
b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java
@@ -33,6 +33,7 @@ import
org.apache.cassandra.simulator.systems.InterceptedWait.InterceptedConditi
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.concurrent.Condition;
import org.apache.cassandra.utils.concurrent.CountDownLatch;
+import org.apache.cassandra.utils.concurrent.Semaphore;
import org.apache.cassandra.utils.concurrent.WaitQueue;
import static
org.apache.cassandra.config.CassandraRelevantProperties.TEST_SIMULATOR_DETERMINISM_CHECK;
@@ -60,6 +61,18 @@ public class InterceptingGlobalMethods extends
InterceptingMonitors implements I
this.onUncaughtException = onUncaughtException
(cassandra) branch cep-15-accord updated: Bring back Journal simulator (w/o Accord at least for now); add semaphore interceptor.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cep-15-accord by this push:
new 72abe53868 Bring back Journal simulator (w/o Accord at least for now);
add semaphore interceptor.
72abe53868 is described below
commit 72abe5386897f8c29b96f953eb8338aa06ee7bda
Author: Alex Petrov
AuthorDate: Wed Jun 5 08:57:05 2024 +0200
Bring back Journal simulator (w/o Accord at least for now); add semaphore
interceptor.
Patch by Alex Petrov; reviewed by David Capwell for CASSANDRA-19695.
---
.../org/apache/cassandra/utils/MonotonicClock.java | 9 +-
.../distributed/test/log/CMSTestBase.java | 9 +-
.../systems/InterceptingGlobalMethods.java | 13 +
.../simulator/systems/InterceptingSemaphore.java | 197
.../systems/InterceptorOfGlobalMethods.java| 27 +-
.../test/AccordJournalSimulationTest.java | 331 ++---
.../cassandra/simulator/test/SemaphoreTest.java| 154 ++
.../simulator/test/SimulationTestBase.java | 36 ++-
.../simulator/test/TrivialSimulationTest.java | 14 +-
9 files changed, 608 insertions(+), 182 deletions(-)
diff --git a/src/java/org/apache/cassandra/utils/MonotonicClock.java
b/src/java/org/apache/cassandra/utils/MonotonicClock.java
index 7be54c008b..c68ed5d9ef 100644
--- a/src/java/org/apache/cassandra/utils/MonotonicClock.java
+++ b/src/java/org/apache/cassandra/utils/MonotonicClock.java
@@ -248,7 +248,14 @@ public interface MonotonicClock
public static class SystemClock extends AbstractEpochSamplingClock
{
-private SystemClock()
+// Without making this constructor public you may start getting the
following exception in the simulator:
+//java.lang.IncompatibleClassChangeError: Type
+// org.apache.cassandra.utils.MonotonicClock$Global is not a
nest member of
+// org.apache.cassandra.utils.MonotonicClock: types are in
different packages
+// There might be a problem with a simulator and how we allow access,
but I verified the change access
+// flags on method of the
org/apache/cassandra/utils/MonotonicClock$SystemClock
+// class to ACC_PUBLIC, and ensured proper testing relationship from
both the surrounding and nested class.
+public SystemClock()
{
super(Clock.Global::currentTimeMillis);
}
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java
b/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java
index c1975cdbfe..e254798915 100644
---
a/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java
+++
b/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java
@@ -91,10 +91,15 @@ public class CMSTestBase
public final TokenPlacementModel.ReplicationFactor rf;
public CMSSut(IIsolatedExecutor.SerializableFunction processorFactory, boolean addListeners,
TokenPlacementModel.ReplicationFactor rf)
+{
+this(processorFactory, addListeners,
Mockito.mock(SchemaProvider.class), rf);
+}
+
+public CMSSut(IIsolatedExecutor.SerializableFunction processorFactory, boolean addListeners, SchemaProvider
schemaProvider, TokenPlacementModel.ReplicationFactor rf)
{
partitioner = Murmur3Partitioner.instance;
this.rf = rf;
-schemaProvider = Mockito.mock(SchemaProvider.class);
+this.schemaProvider = schemaProvider;
ClusterMetadata initial = new ClusterMetadata(partitioner);
log = LocalLog.logSpec()
.sync()
@@ -127,7 +132,7 @@ public class CMSTestBase
}, schemaProvider));
}
-public void close() throws Exception
+public void close()
{
ClusterMetadataService.unsetInstance();
}
diff --git
a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java
b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java
index abc71eae35..34c0f6bacc 100644
---
a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java
+++
b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java
@@ -33,6 +33,7 @@ import
org.apache.cassandra.simulator.systems.InterceptedWait.InterceptedConditi
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.concurrent.Condition;
import org.apache.cassandra.utils.concurrent.CountDownLatch;
+import org.apache.cassandra.utils.concurrent.Semaphore;
import org.apache.cassandra.utils.concurrent.WaitQueue;
import static
org.apache.cassandra.config.CassandraRelevantProperties.TEST_SIMULATOR_DETERMINISM_CHECK
(cassandra) branch cep-15-accord updated: Accord Journal Determinism: PreAccept replay stability
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cep-15-accord by this push:
new b0ca509e7a Accord Journal Determinism: PreAccept replay stability
b0ca509e7a is described below
commit b0ca509e7add760d187fcc5a9908d93d7c4fd6ec
Author: Alex Petrov
AuthorDate: Wed May 29 19:16:26 2024 +0200
Accord Journal Determinism: PreAccept replay stability
Patch by Alex Petrov; reviewed by Aleksey Yeschenko for CASSANDRA-19664
---
modules/accord | 2 +-
.../apache/cassandra/journal/RecordPointer.java| 66 +++
.../service/accord/AccordCommandStore.java | 4 +-
.../cassandra/service/accord/AccordJournal.java| 206 +
.../service/accord/AccordSafeCommandStore.java | 49 -
5 files changed, 197 insertions(+), 130 deletions(-)
diff --git a/modules/accord b/modules/accord
index 4e8bcae81f..84e89bd91c 16
--- a/modules/accord
+++ b/modules/accord
@@ -1 +1 @@
-Subproject commit 4e8bcae81f9751b9d732fd5056bce31c97ad58f3
+Subproject commit 84e89bd91cf1b058fbf314b750336a1ec1096b18
diff --git a/src/java/org/apache/cassandra/journal/RecordPointer.java
b/src/java/org/apache/cassandra/journal/RecordPointer.java
new file mode 100644
index 00..2b3e8ea6b8
--- /dev/null
+++ b/src/java/org/apache/cassandra/journal/RecordPointer.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.journal;
+
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+
+// TODO: make this available in the accord table as an ID
+public class RecordPointer implements Comparable
+{
+public final long segment; // unique segment id
+public final int position; // record start position within the segment
+
+public RecordPointer(long segment, int position)
+{
+this.segment = segment;
+this.position = position;
+}
+
+@Override
+public boolean equals(Object other)
+{
+if (this == other)
+return true;
+if (!(other instanceof RecordPointer))
+return false;
+RecordPointer that = (RecordPointer) other;
+return this.segment == that.segment
+ && this.position == that.position;
+}
+
+@Override
+public int hashCode()
+{
+return Long.hashCode(segment) + position * 31;
+}
+
+@Override
+public String toString()
+{
+return "(" + segment + ", " + position + ')';
+}
+
+@Override
+public int compareTo(RecordPointer that)
+{
+int cmp = Longs.compare(this.segment, that.segment);
+return cmp != 0 ? cmp : Ints.compare(this.position, that.position);
+}
+}
\ No newline at end of file
diff --git
a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
index 2a67ba656d..c846038fd8 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
@@ -488,7 +488,9 @@ public class AccordCommandStore extends CommandStore
implements CacheSize
timestampsForKeys.values().forEach(AccordSafeState::preExecute);
if (commandsForRanges != null)
commandsForRanges.preExecute();
-current = new AccordSafeCommandStore(preLoadContext, commands,
timestampsForKeys, commandsForKeys, commandsForRanges, this);
+
+current = AccordSafeCommandStore.create(preLoadContext, commands,
timestampsForKeys, commandsForKeys, commandsForRanges, this);
+
return current;
}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordJournal.java
b/src/java/org/apache/cassandra/service/accord/AccordJournal.java
index ce90b26747..0c31afbb4c 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordJournal.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordJournal.java
@@ -40,
(cassandra-accord) branch trunk updated: Accord Journal Determinism: PreAccept replay stability
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
The following commit(s) were added to refs/heads/trunk by this push:
new 84e89bd9 Accord Journal Determinism: PreAccept replay stability
84e89bd9 is described below
commit 84e89bd91cf1b058fbf314b750336a1ec1096b18
Author: Alex Petrov
AuthorDate: Wed May 29 14:32:33 2024 +0200
Accord Journal Determinism: PreAccept replay stability
Patch by Alex Petrov; reviewed by Aleksey Yeschenko for CASSANDRA-19664
---
.../src/main/java/accord/local/CommandStore.java | 6 +
.../main/java/accord/local/SafeCommandStore.java | 5
.../java/accord/messages/ExecutionContext.java | 29 --
.../src/main/java/accord/messages/Propagate.java | 1 -
.../main/java/accord/messages/ReplyContext.java| 1 +
.../src/main/java/accord/messages/TxnRequest.java | 5
6 files changed, 12 insertions(+), 35 deletions(-)
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java
b/accord-core/src/main/java/accord/local/CommandStore.java
index 1d5a3913..c7baf2f5 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -319,13 +319,9 @@ public abstract class CommandStore implements AgentExecutor
*/
final Timestamp preaccept(TxnId txnId, Seekables keys,
SafeCommandStore safeStore, boolean permitFastPath)
{
-// TODO (expected): make preAcceptTimeout() be a part of
SafeCommandStore, initiated from ExecutionContext;
-// preAcceptTimeout can be subject to local configuration
changes, which would break determinism of repeated
-// message processing, if, say, replayed from a log.
-
NodeTimeService time = safeStore.time();
-boolean isExpired = time.now() - txnId.hlc() >=
agent().preAcceptTimeout() && !txnId.kind().isSyncPoint();
+boolean isExpired = time.now() - txnId.hlc() >=
safeStore.preAcceptTimeout() && !txnId.kind().isSyncPoint();
if (rejectBefore != null && !isExpired)
isExpired = null == rejectBefore.foldl(keys, (rejectIfBefore,
test) -> rejectIfBefore.compareTo(test) > 0 ? null : test, txnId,
Objects::isNull);
diff --git a/accord-core/src/main/java/accord/local/SafeCommandStore.java
b/accord-core/src/main/java/accord/local/SafeCommandStore.java
index 5c8e2834..139bf088 100644
--- a/accord-core/src/main/java/accord/local/SafeCommandStore.java
+++ b/accord-core/src/main/java/accord/local/SafeCommandStore.java
@@ -183,6 +183,11 @@ public abstract class SafeCommandStore
return maybeTruncate(safeCfk);
}
+public long preAcceptTimeout()
+{
+return agent().preAcceptTimeout();
+}
+
protected abstract SafeCommand getInternal(TxnId txnId);
protected abstract SafeCommand getInternalIfLoadedAndInitialised(TxnId
txnId);
protected abstract SafeCommandsForKey getInternal(Key key);
diff --git a/accord-core/src/main/java/accord/messages/ExecutionContext.java
b/accord-core/src/main/java/accord/messages/ExecutionContext.java
deleted file mode 100644
index dbf4c2db..
--- a/accord-core/src/main/java/accord/messages/ExecutionContext.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package accord.messages;
-
-/**
- * Necessary context to allow for deterministic repeated execution of requests
(e.g. when re-applying from a log)
- */
-public interface ExecutionContext
-{
-/**
- * @return PreAccept timeout as it was at request execution
- */
-long preAcceptTimeout();
-}
diff --git a/accord-core/src/main/java/accord/messages/Propagate.java
b/accord-core/src/main/java/accord/messages/Propagate.java
index 351c636e..c67d5fa2 100644
--- a/accord-core/src/main/java/accord/messages/Propagate.java
+++ b/accord-core/src/main/java/accord/messages/Propagate.java
@@ -53,7 +53,6 @@ import static accord.local.SaveStatus.Stable;
import static accord.local.SaveStatus.Uninitialised;
impo
(cassandra) branch trunk updated (8ba2f9e8c0 -> d10008d54b)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git from 8ba2f9e8c0 Consolidate logging on trace level add 9ebe0aa08a Replace getStderr calls with getCleanedStderr calls in tests checking for emptiness add dc17c29724 Add native transport deadline, an ultimate deadline for all tasks related to a specific request add 617a75843c Merge branch 'cassandra-4.1' into cassandra-5.0 add d10008d54b Merge branch 'cassandra-5.0' into trunk No new revisions were added by this update. Summary of changes: CHANGES.txt| 2 +- .../cassandra/auth/CIDRGroupsMappingManager.java | 5 +- .../cassandra/auth/CIDRPermissionsManager.java | 4 +- .../apache/cassandra/auth/CassandraAuthorizer.java | 7 +- .../cassandra/auth/CassandraNetworkAuthorizer.java | 4 +- .../cassandra/auth/CassandraRoleManager.java | 27 +- .../cassandra/auth/PasswordAuthenticator.java | 4 +- .../apache/cassandra/batchlog/BatchlogManager.java | 8 +- .../cassandra/concurrent/DebuggableTask.java | 9 + .../apache/cassandra/concurrent/FutureTask.java| 28 ++ .../cassandra/concurrent/ResizableThreadPool.java | 5 + .../apache/cassandra/concurrent/SEPExecutor.java | 15 + src/java/org/apache/cassandra/config/Config.java | 20 ++ .../cassandra/config/DatabaseDescriptor.java | 103 ++ .../org/apache/cassandra/cql3/CQLStatement.java| 5 +- .../cql3/CustomPayloadMirroringQueryHandler.java | 16 +- .../org/apache/cassandra/cql3/QueryHandler.java| 7 +- .../org/apache/cassandra/cql3/QueryProcessor.java | 42 +-- .../apache/cassandra/cql3/UntypedResultSet.java| 5 +- .../cql3/statements/AuthenticationStatement.java | 5 +- .../cql3/statements/AuthorizationStatement.java| 4 +- .../cassandra/cql3/statements/BatchStatement.java | 34 +- .../cql3/statements/DescribeStatement.java | 3 +- .../cql3/statements/ModificationStatement.java | 50 +-- .../cassandra/cql3/statements/SelectStatement.java | 30 +- .../cql3/statements/TruncateStatement.java | 4 +- .../cassandra/cql3/statements/UseStatement.java| 7 +- .../statements/schema/AlterSchemaStatement.java| 4 +- .../cassandra/db/CounterMutationVerbHandler.java | 6 +- .../apache/cassandra/db/MutationVerbHandler.java | 9 + .../cassandra/db/PartitionRangeReadCommand.java| 7 +- src/java/org/apache/cassandra/db/ReadCommand.java | 20 +- src/java/org/apache/cassandra/db/ReadQuery.java| 6 +- .../cassandra/db/SinglePartitionReadCommand.java | 17 +- .../org/apache/cassandra/db/view/TableViews.java | 5 +- .../apache/cassandra/db/view/ViewBuilderTask.java | 4 +- .../db/virtual/CIDRFilteringMetricsTable.java | 4 +- .../locator/AbstractReplicationStrategy.java | 15 +- .../apache/cassandra/metrics/ClientMetrics.java| 32 +- .../cassandra/metrics/ThreadPoolMetrics.java | 7 + .../cassandra/net/InboundMessageHandler.java | 12 +- src/java/org/apache/cassandra/net/Message.java | 45 ++- .../apache/cassandra/repair/RepairCoordinator.java | 3 +- .../service/AbstractWriteResponseHandler.java | 43 ++- .../cassandra/service/BatchlogResponseHandler.java | 5 +- .../apache/cassandra/service/CassandraDaemon.java | 16 +- .../DatacenterSyncWriteResponseHandler.java| 5 +- .../service/DatacenterWriteResponseHandler.java| 5 +- .../cassandra/service/NativeTransportService.java | 7 +- .../org/apache/cassandra/service/StorageProxy.java | 346 +++-- .../apache/cassandra/service/StorageService.java | 71 - .../cassandra/service/StorageServiceMBean.java | 19 +- .../cassandra/service/WriteResponseHandler.java| 9 +- .../service/pager/AbstractQueryPager.java | 5 +- .../service/pager/AggregationQueryPager.java | 41 ++- .../service/pager/MultiPartitionPager.java | 19 +- .../apache/cassandra/service/pager/QueryPager.java | 5 +- .../org/apache/cassandra/service/paxos/Paxos.java | 18 +- .../service/paxos/v1/AbstractPaxosCallback.java| 16 +- .../service/paxos/v1/PrepareCallback.java | 5 +- .../service/paxos/v1/ProposeCallback.java | 5 +- .../service/reads/AbstractReadExecutor.java| 70 +++-- .../cassandra/service/reads/DataResolver.java | 13 +- .../cassandra/service/reads/DigestResolver.java| 7 +- .../cassandra/service/reads/ReadCallback.java | 42 ++- .../service/reads/ReplicaFilteringProtection.java | 15 +- .../cassandra/service/reads/ResponseResolver.java | 7 +- .../reads/ShortReadPartitionsProtection.java | 15 +- .../service/reads/ShortReadProtection.java | 5 +- .../service/reads/range/RangeCommandIterator.java | 26 +- .../service/reads/rang
(cassandra) branch cassandra-4.1 updated (aa20c9ab11 -> dc17c29724)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cassandra-4.1 in repository https://gitbox.apache.org/repos/asf/cassandra.git from aa20c9ab11 Merge branch 'cassandra-4.0' into cassandra-4.1 add 9ebe0aa08a Replace getStderr calls with getCleanedStderr calls in tests checking for emptiness add dc17c29724 Add native transport deadline, an ultimate deadline for all tasks related to a specific request No new revisions were added by this update. Summary of changes: CHANGES.txt| 2 +- .../apache/cassandra/auth/CassandraAuthorizer.java | 8 +- .../cassandra/auth/CassandraNetworkAuthorizer.java | 5 +- .../cassandra/auth/CassandraRoleManager.java | 5 +- .../cassandra/auth/PasswordAuthenticator.java | 5 +- .../apache/cassandra/batchlog/BatchlogManager.java | 8 +- .../cassandra/concurrent/DebuggableTask.java | 9 + .../apache/cassandra/concurrent/FutureTask.java| 28 ++ .../cassandra/concurrent/ResizableThreadPool.java | 5 + .../apache/cassandra/concurrent/SEPExecutor.java | 15 + src/java/org/apache/cassandra/config/Config.java | 16 + .../cassandra/config/DatabaseDescriptor.java | 96 +- .../org/apache/cassandra/cql3/CQLStatement.java| 5 +- .../cql3/CustomPayloadMirroringQueryHandler.java | 16 +- .../org/apache/cassandra/cql3/QueryHandler.java| 7 +- .../org/apache/cassandra/cql3/QueryProcessor.java | 42 +-- .../apache/cassandra/cql3/UntypedResultSet.java| 5 +- .../cql3/statements/AuthenticationStatement.java | 5 +- .../cql3/statements/AuthorizationStatement.java| 4 +- .../cassandra/cql3/statements/BatchStatement.java | 34 +- .../cql3/statements/DescribeStatement.java | 3 +- .../cql3/statements/ModificationStatement.java | 50 +-- .../cassandra/cql3/statements/SelectStatement.java | 30 +- .../cql3/statements/TruncateStatement.java | 4 +- .../cassandra/cql3/statements/UseStatement.java| 7 +- .../statements/schema/AlterSchemaStatement.java| 4 +- .../cassandra/db/CounterMutationVerbHandler.java | 6 +- .../apache/cassandra/db/MutationVerbHandler.java | 9 + .../cassandra/db/PartitionRangeReadCommand.java| 7 +- src/java/org/apache/cassandra/db/ReadCommand.java | 20 +- src/java/org/apache/cassandra/db/ReadQuery.java| 5 +- .../cassandra/db/SinglePartitionReadCommand.java | 17 +- .../org/apache/cassandra/db/view/TableViews.java | 5 +- .../apache/cassandra/db/view/ViewBuilderTask.java | 4 +- .../locator/AbstractReplicationStrategy.java | 15 +- .../apache/cassandra/metrics/ClientMetrics.java| 45 ++- .../cassandra/metrics/ThreadPoolMetrics.java | 7 + .../cassandra/net/InboundMessageHandler.java | 12 +- src/java/org/apache/cassandra/net/Message.java | 31 +- .../apache/cassandra/repair/RepairRunnable.java| 4 +- .../service/AbstractWriteResponseHandler.java | 20 +- .../cassandra/service/BatchlogResponseHandler.java | 5 +- .../DatacenterSyncWriteResponseHandler.java| 5 +- .../service/DatacenterWriteResponseHandler.java| 5 +- .../org/apache/cassandra/service/StorageProxy.java | 343 +++-- .../apache/cassandra/service/StorageService.java | 79 + .../cassandra/service/StorageServiceMBean.java | 19 ++ .../cassandra/service/WriteResponseHandler.java| 9 +- .../service/pager/AbstractQueryPager.java | 5 +- .../service/pager/AggregationQueryPager.java | 41 ++- .../service/pager/MultiPartitionPager.java | 18 +- .../apache/cassandra/service/pager/QueryPager.java | 5 +- .../org/apache/cassandra/service/paxos/Paxos.java | 18 +- .../service/paxos/v1/AbstractPaxosCallback.java| 13 +- .../service/paxos/v1/PrepareCallback.java | 5 +- .../service/paxos/v1/ProposeCallback.java | 5 +- .../service/reads/AbstractReadExecutor.java| 71 +++-- .../cassandra/service/reads/DataResolver.java | 13 +- .../cassandra/service/reads/DigestResolver.java| 7 +- .../cassandra/service/reads/ReadCallback.java | 31 +- .../service/reads/ReplicaFilteringProtection.java | 15 +- .../cassandra/service/reads/ResponseResolver.java | 7 +- .../reads/ShortReadPartitionsProtection.java | 15 +- .../service/reads/ShortReadProtection.java | 5 +- .../service/reads/range/RangeCommandIterator.java | 24 +- .../service/reads/range/RangeCommands.java | 9 +- .../service/reads/repair/AbstractReadRepair.java | 20 +- .../service/reads/repair/BlockingReadRepair.java | 9 +- .../service/reads/repair/ReadOnlyReadRepair.java | 5 +- .../cassandra/service/reads/repair/ReadRepair.java | 7 +- .../service/reads/repair/ReadRepairStrategy.java | 9 +- .../apache/cassandra/tracing/TraceStateImpl.java | 4 +- .../cassandra
(cassandra) branch cassandra-5.0 updated (fbfa77e70f -> 617a75843c)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cassandra-5.0 in repository https://gitbox.apache.org/repos/asf/cassandra.git from fbfa77e70f Merge branch 'cassandra-4.1' into cassandra-5.0 add 9ebe0aa08a Replace getStderr calls with getCleanedStderr calls in tests checking for emptiness add dc17c29724 Add native transport deadline, an ultimate deadline for all tasks related to a specific request add 617a75843c Merge branch 'cassandra-4.1' into cassandra-5.0 No new revisions were added by this update. Summary of changes: CHANGES.txt| 2 +- .../cassandra/auth/CIDRGroupsMappingManager.java | 5 +- .../cassandra/auth/CIDRPermissionsManager.java | 4 +- .../apache/cassandra/auth/CassandraAuthorizer.java | 7 +- .../cassandra/auth/CassandraNetworkAuthorizer.java | 4 +- .../cassandra/auth/CassandraRoleManager.java | 4 +- .../cassandra/auth/PasswordAuthenticator.java | 4 +- .../apache/cassandra/batchlog/BatchlogManager.java | 8 +- .../cassandra/concurrent/DebuggableTask.java | 9 + .../apache/cassandra/concurrent/FutureTask.java| 28 ++ .../cassandra/concurrent/ResizableThreadPool.java | 5 + .../apache/cassandra/concurrent/SEPExecutor.java | 15 + src/java/org/apache/cassandra/config/Config.java | 20 ++ .../cassandra/config/DatabaseDescriptor.java | 103 ++ .../org/apache/cassandra/cql3/CQLStatement.java| 5 +- .../cql3/CustomPayloadMirroringQueryHandler.java | 16 +- .../org/apache/cassandra/cql3/QueryHandler.java| 7 +- .../org/apache/cassandra/cql3/QueryProcessor.java | 42 +-- .../apache/cassandra/cql3/UntypedResultSet.java| 5 +- .../cql3/statements/AuthenticationStatement.java | 5 +- .../cql3/statements/AuthorizationStatement.java| 4 +- .../cassandra/cql3/statements/BatchStatement.java | 34 +- .../cql3/statements/DescribeStatement.java | 3 +- .../cql3/statements/ModificationStatement.java | 50 +-- .../cassandra/cql3/statements/SelectStatement.java | 30 +- .../cql3/statements/TruncateStatement.java | 4 +- .../cassandra/cql3/statements/UseStatement.java| 7 +- .../statements/schema/AlterSchemaStatement.java| 4 +- .../cassandra/db/CounterMutationVerbHandler.java | 6 +- .../apache/cassandra/db/MutationVerbHandler.java | 9 + .../cassandra/db/PartitionRangeReadCommand.java| 7 +- src/java/org/apache/cassandra/db/ReadCommand.java | 20 +- src/java/org/apache/cassandra/db/ReadQuery.java| 6 +- .../cassandra/db/SinglePartitionReadCommand.java | 17 +- .../org/apache/cassandra/db/view/TableViews.java | 5 +- .../apache/cassandra/db/view/ViewBuilderTask.java | 4 +- .../db/virtual/CIDRFilteringMetricsTable.java | 4 +- .../locator/AbstractReplicationStrategy.java | 15 +- .../apache/cassandra/metrics/ClientMetrics.java| 44 ++- .../cassandra/metrics/ThreadPoolMetrics.java | 7 + .../cassandra/net/InboundMessageHandler.java | 12 +- src/java/org/apache/cassandra/net/Message.java | 31 +- .../apache/cassandra/repair/RepairCoordinator.java | 3 +- .../service/AbstractWriteResponseHandler.java | 41 ++- .../cassandra/service/BatchlogResponseHandler.java | 5 +- .../apache/cassandra/service/CassandraDaemon.java | 16 +- .../DatacenterSyncWriteResponseHandler.java| 5 +- .../service/DatacenterWriteResponseHandler.java| 5 +- .../cassandra/service/NativeTransportService.java | 7 +- .../org/apache/cassandra/service/StorageProxy.java | 348 +++-- .../apache/cassandra/service/StorageService.java | 71 - .../cassandra/service/StorageServiceMBean.java | 21 +- .../cassandra/service/WriteResponseHandler.java| 9 +- .../service/pager/AbstractQueryPager.java | 5 +- .../service/pager/AggregationQueryPager.java | 41 ++- .../service/pager/MultiPartitionPager.java | 19 +- .../apache/cassandra/service/pager/QueryPager.java | 5 +- .../org/apache/cassandra/service/paxos/Paxos.java | 18 +- .../service/paxos/v1/AbstractPaxosCallback.java| 16 +- .../service/paxos/v1/PrepareCallback.java | 5 +- .../service/paxos/v1/ProposeCallback.java | 5 +- .../service/reads/AbstractReadExecutor.java| 73 +++-- .../cassandra/service/reads/DataResolver.java | 13 +- .../cassandra/service/reads/DigestResolver.java| 7 +- .../cassandra/service/reads/ReadCallback.java | 40 ++- .../service/reads/ReplicaFilteringProtection.java | 15 +- .../cassandra/service/reads/ResponseResolver.java | 7 +- .../reads/ShortReadPartitionsProtection.java | 15 +- .../service/reads/ShortReadProtection.java | 5 +- .../service/reads/range/RangeCommandIterator.java | 26 +- .../service/reads/range/RangeCommands.java | 11
(cassandra) branch trunk updated: Reuse native transport-driven futures in Debounce.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2e05cd4c8d Reuse native transport-driven futures in Debounce.
2e05cd4c8d is described below
commit 2e05cd4c8dd22e458eb1d2dad9cd34936b470266
Author: Alex Petrov
AuthorDate: Tue May 28 16:55:58 2024 +0200
Reuse native transport-driven futures in Debounce.
Patch by Alex Petrov; reviewed by Sam Tunnicliffe for CASSANDRA-19158.
---
.../cassandra/auth/CassandraRoleManager.java | 28 ---
.../config/CassandraRelevantProperties.java| 1 +
.../apache/cassandra/gms/GossipVerbHandler.java| 2 +
.../apache/cassandra/service/StorageService.java | 11 ++-
.../apache/cassandra/tcm/EpochAwareDebounce.java | 98 --
.../org/apache/cassandra/tcm/PeerLogFetcher.java | 48 +++
.../org/apache/cassandra/tcm/RemoteProcessor.java | 49 ++-
.../distributed/impl/AbstractCluster.java | 11 ++-
.../cassandra/distributed/impl/Instance.java | 6 +-
.../cassandra/distributed/test/GossipTest.java | 1 +
.../test/HintedHandoffAddRemoveNodesTest.java | 8 +-
.../distributed/test/log/CMSCatchupTest.java | 29 ---
.../test/log/FetchLogFromPeersTest.java| 24 +-
.../distributed/test/ring/DecommissionTest.java| 5 +-
14 files changed, 180 insertions(+), 141 deletions(-)
diff --git a/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
b/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
index 921a45dcb2..a046adc2a9 100644
--- a/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
+++ b/src/java/org/apache/cassandra/auth/CassandraRoleManager.java
@@ -137,11 +137,11 @@ public class CassandraRoleManager implements IRoleManager
public CassandraRoleManager()
{
supportedOptions = DatabaseDescriptor.getAuthenticator() instanceof
PasswordAuthenticator
- ? ImmutableSet.of(Option.LOGIN, Option.SUPERUSER,
Option.PASSWORD, Option.HASHED_PASSWORD)
- : ImmutableSet.of(Option.LOGIN, Option.SUPERUSER);
+ ? ImmutableSet.of(Option.LOGIN, Option.SUPERUSER,
Option.PASSWORD, Option.HASHED_PASSWORD)
+ : ImmutableSet.of(Option.LOGIN, Option.SUPERUSER);
alterableOptions = DatabaseDescriptor.getAuthenticator() instanceof
PasswordAuthenticator
- ? ImmutableSet.of(Option.PASSWORD,
Option.HASHED_PASSWORD)
- : ImmutableSet.of();
+ ? ImmutableSet.of(Option.PASSWORD,
Option.HASHED_PASSWORD)
+ : ImmutableSet.of();
}
@Override
@@ -149,17 +149,23 @@ public class CassandraRoleManager implements IRoleManager
{
loadRoleStatement();
loadIdentityStatement();
-if (asyncRoleSetup)
+if (!asyncRoleSetup)
{
-scheduleSetupTask(() -> {
+try
+{
+// Try to set up synchronously
setupDefaultRole();
-return null;
-});
+return;
+}
+catch (Throwable t)
+{
+// We tried to execute the task in a sync way, but failed. Try
asynchronous setup.
+}
}
-else
-{
+scheduleSetupTask(() -> {
setupDefaultRole();
-}
+return null;
+});
}
@Override
diff --git
a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index 57f442d1d1..e1f6d28d86 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -474,6 +474,7 @@ public enum CassandraRelevantProperties
SET_SEP_THREAD_NAME("cassandra.set_sep_thread_name", "true"),
SHUTDOWN_ANNOUNCE_DELAY_IN_MS("cassandra.shutdown_announce_in_ms", "2000"),
SIZE_RECORDER_INTERVAL("cassandra.size_recorder_interval", "300"),
+SKIP_AUTH_SETUP("cassandra.skip_auth_setup", "false"),
SKIP_GC_INSPECTOR("cassandra.skip_gc_inspector", "false"),
SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE("cassandra.skip_paxos_repair_on_topology_change"),
/** If necessary for operational purposes, permit certain keyspaces to be
ignored for paxos topology repairs. */
diff --git a/src/java/org/apache/cassandra/gms/GossipVerbHandler.java
b/src/java/org/apache/cassandra/gms/GossipVerbHandler.java
index 02aeaf4467..bac0854d07 100644
--- a/src/java/org/apache/cassandra/gms/GossipVerbHandler.java
+++ b/src/
(cassandra) branch trunk updated: Fix peers v2 system table behaviour when 2 nodes swap their IP Addresses
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 38512a469c Fix peers v2 system table behaviour when 2 nodes swap their
IP Addresses
38512a469c is described below
commit 38512a469cef06770384423d0b30e3e85b511258
Author: Alex Petrov
AuthorDate: Mon Apr 8 13:04:07 2024 +0200
Fix peers v2 system table behaviour when 2 nodes swap their IP Addresses
Throw if node id has been changed and does not match directory. If, however
the _ip_ address has changed, issue Startup and correct the IP address.
Disallow picking over identity of other nodes via hijacking their IPs or via
overriding local node id with theirs.
Patch by Alex Petrov; reviewed by Sam Tunnicliffe for CASSANDRA-19221
---
.../apache/cassandra/db/virtual/PeersTable.java| 26 ++---
src/java/org/apache/cassandra/tcm/Startup.java | 13 ++-
.../tcm/listeners/LegacyStateListener.java | 10 +-
.../test/log/BounceResetHostIdTest.java| 117 ++---
4 files changed, 127 insertions(+), 39 deletions(-)
diff --git a/src/java/org/apache/cassandra/db/virtual/PeersTable.java
b/src/java/org/apache/cassandra/db/virtual/PeersTable.java
index 5b011de604..8d50dd7e90 100644
--- a/src/java/org/apache/cassandra/db/virtual/PeersTable.java
+++ b/src/java/org/apache/cassandra/db/virtual/PeersTable.java
@@ -113,15 +113,6 @@ public class PeersTable extends AbstractVirtualTable
return result;
}
-public static void initializeLegacyPeerTables(ClusterMetadata prev,
ClusterMetadata next)
-{
-QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s",
SYSTEM_KEYSPACE_NAME, PEERS_V2));
-QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s",
SYSTEM_KEYSPACE_NAME, LEGACY_PEERS));
-
-for (NodeId nodeId : next.directory.peerIds())
-updateLegacyPeerTable(nodeId, prev, next);
-}
-
private static String peers_v2_query = "INSERT INTO %s.%s ("
+ "peer, peer_port, "
+ "preferred_ip, preferred_port, "
@@ -156,9 +147,7 @@ public class PeersTable extends AbstractVirtualTable
if (next.directory.peerState(nodeId) == null ||
next.directory.peerState(nodeId) == NodeState.LEFT)
{
NodeAddresses addresses = prev.directory.getNodeAddresses(nodeId);
-logger.debug("Purging {} from system.peers_v2 table", addresses);
-QueryProcessor.executeInternal(String.format(peers_delete_query,
SYSTEM_KEYSPACE_NAME, PEERS_V2), addresses.broadcastAddress.getAddress(),
addresses.broadcastAddress.getPort());
-
QueryProcessor.executeInternal(String.format(legacy_peers_delete_query,
SYSTEM_KEYSPACE_NAME, LEGACY_PEERS), addresses.broadcastAddress.getAddress());
+removeFromSystemPeersTables(addresses.broadcastAddress);
}
else if (NodeState.isPreJoin(next.directory.peerState(nodeId)))
{
@@ -169,11 +158,7 @@ public class PeersTable extends AbstractVirtualTable
NodeAddresses addresses = next.directory.getNodeAddresses(nodeId);
NodeAddresses oldAddresses =
prev.directory.getNodeAddresses(nodeId);
if (oldAddresses != null && !oldAddresses.equals(addresses))
-{
-logger.debug("Purging {} from system.peers_v2 table",
oldAddresses);
-
QueryProcessor.executeInternal(String.format(peers_delete_query,
SYSTEM_KEYSPACE_NAME, PEERS_V2), oldAddresses.broadcastAddress.getAddress(),
oldAddresses.broadcastAddress.getPort());
-
QueryProcessor.executeInternal(String.format(legacy_peers_delete_query,
SYSTEM_KEYSPACE_NAME, LEGACY_PEERS),
oldAddresses.broadcastAddress.getAddress());
-}
+removeFromSystemPeersTables(oldAddresses.broadcastAddress);
Location location = next.directory.location(nodeId);
@@ -197,4 +182,11 @@ public class PeersTable extends AbstractVirtualTable
tokens);
}
}
+
+public static void removeFromSystemPeersTables(InetAddressAndPort addr)
+{
+logger.debug("Purging {} from system.peers_v2 table", addr);
+QueryProcessor.executeInternal(String.format(peers_delete_query,
SYSTEM_KEYSPACE_NAME, PEERS_V2), addr.getAddress(), addr.getPort());
+
QueryProcessor.executeInternal(String.format(legacy_peers_delete_query,
SYSTEM_KEYSPACE_NAME, LEGACY_PEERS), addr.getAddress());
+}
}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tcm/Startup.java
b/src/java/org/apache/cassandra/tcm/Startup.java
index dfe4df8399..023fcdba77 100644
---
(cassandra) branch trunk updated: The result of applying a metadata snapshot via ForceSnapshot should return the correct set of modified keys
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7623e4678b The result of applying a metadata snapshot via
ForceSnapshot should return the correct set of modified keys
7623e4678b is described below
commit 7623e4678b8ef131434f1de3522c6425c092dff9
Author: Alex Petrov
AuthorDate: Mon Mar 25 10:25:50 2024 +0100
The result of applying a metadata snapshot via ForceSnapshot should return
the correct set of modified keys
Patch by Alex Petrov; reviewed by Marcus Eriksson for CASSANDRA-19128.
---
.../org/apache/cassandra/tcm/MetadataKeys.java | 47 +
.../cassandra/tcm/ownership/PlacementDeltas.java | 22 +++
.../tcm/transformations/CustomTransformation.java | 44 +
.../tcm/transformations/ForceSnapshot.java | 4 +-
.../cassandra/tcm/transformations/PrepareMove.java | 6 +
.../test/log/ClusterMetadataTestHelper.java| 3 +-
.../distributed/test/log/MetadataKeysTest.java | 220 +
.../apache/cassandra/harry/gen/EntropySource.java | 5 +-
.../org/apache/cassandra/harry/gen/Generators.java | 42 +++-
.../org/apache/cassandra/tcm/log/LocalLogTest.java | 10 +-
10 files changed, 388 insertions(+), 15 deletions(-)
diff --git a/src/java/org/apache/cassandra/tcm/MetadataKeys.java
b/src/java/org/apache/cassandra/tcm/MetadataKeys.java
index fda509186b..8028007815 100644
--- a/src/java/org/apache/cassandra/tcm/MetadataKeys.java
+++ b/src/java/org/apache/cassandra/tcm/MetadataKeys.java
@@ -18,10 +18,17 @@
package org.apache.cassandra.tcm;
+import java.util.HashSet;
import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
import com.google.common.collect.ImmutableSet;
+import org.apache.cassandra.tcm.extensions.ExtensionKey;
+import org.apache.cassandra.tcm.extensions.ExtensionValue;
+
public class MetadataKeys
{
public static final String CORE_NS =
MetadataKeys.class.getPackage().getName().toLowerCase(Locale.ROOT);
@@ -52,4 +59,44 @@ public class MetadataKeys
return new MetadataKey(b.toString());
}
+public static ImmutableSet diffKeys(ClusterMetadata before,
ClusterMetadata after)
+{
+ImmutableSet.Builder builder = new
ImmutableSet.Builder<>();
+diffKeys(before, after, builder);
+return builder.build();
+}
+
+private static void diffKeys(ClusterMetadata before, ClusterMetadata
after, ImmutableSet.Builder builder)
+{
+checkKey(before, after, builder, cm -> cm.schema, MetadataKeys.SCHEMA);
+checkKey(before, after, builder, cm -> cm.directory,
MetadataKeys.NODE_DIRECTORY);
+checkKey(before, after, builder, cm -> cm.tokenMap,
MetadataKeys.TOKEN_MAP);
+checkKey(before, after, builder, cm -> cm.placements,
MetadataKeys.DATA_PLACEMENTS);
+checkKey(before, after, builder, cm -> cm.lockedRanges,
MetadataKeys.LOCKED_RANGES);
+checkKey(before, after, builder, cm -> cm.inProgressSequences,
MetadataKeys.IN_PROGRESS_SEQUENCES);
+
+Set> added = new
HashSet<>(after.extensions.keySet());
+for (Map.Entry, ExtensionValue> entry :
before.extensions.entrySet())
+{
+ExtensionKey key = entry.getKey();
+added.remove(key);
+
+if (after.extensions.containsKey(key))
+checkKey(before, after, builder, cm -> cm.extensions.get(key),
key);
+else
+builder.add(key);
+}
+
+for (ExtensionKey key : added)
+builder.add(key);
+}
+
+private static void checkKey(ClusterMetadata before, ClusterMetadata
after, ImmutableSet.Builder builder, Function> extract, MetadataKey key)
+{
+MetadataValue vBefore = extract.apply(before);
+MetadataValue vAfter = extract.apply(after);
+
+if (!vBefore.equals(vAfter))
+builder.add(key);
+}
}
diff --git a/src/java/org/apache/cassandra/tcm/ownership/PlacementDeltas.java
b/src/java/org/apache/cassandra/tcm/ownership/PlacementDeltas.java
index 6b5b817984..6ba80a854b 100644
--- a/src/java/org/apache/cassandra/tcm/ownership/PlacementDeltas.java
+++ b/src/java/org/apache/cassandra/tcm/ownership/PlacementDeltas.java
@@ -77,6 +77,28 @@ public class PlacementDeltas extends
ReplicationMap e : map.entrySet())
+{
+if (!e.getValue().reads.removals.isEmpty())
+return false;
+if (!e.getValue().reads.additions.isEmpty())
+return false;
+
+if (!e.getValue().writes.removals.isEmpty())
+return false;
+if (!e.getValue().writes.additions.isEmpty())
+return false;
+}
+
+return true;
+}
+
public static PlacementDeltas
(cassandra) branch trunk updated: Harry model that supports value overrides: an ability to provide specific values for clustering, regular, and static columns
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6b48f8a11d Harry model that supports value overrides: an ability to
provide specific values for clustering, regular, and static columns
6b48f8a11d is described below
commit 6b48f8a11dbad8c0653309eb8193fa6157bba5d8
Author: Alex Petrov
AuthorDate: Wed Jan 17 19:12:43 2024 +0100
Harry model that supports value overrides: an ability to provide specific
values for clustering, regular, and static columns
Patch by Alex Petrov; reviewed by Caleb Rackliffe for CASSANDRA-19284
---
.../cassandra/distributed/shared/ClusterUtils.java | 2 +
.../fuzz/harry/examples/RepairBurnTest.java| 138
.../dsl/HistoryBuilderIntegrationTest.java | 153 ++---
.../HistoryBuilderOverridesIntegrationTest.java| 359 +
.../integration/model/IntegrationTestBase.java | 12 +-
.../model/ReconcilerIntegrationTest.java | 23 +-
.../fuzz/ring/ConsistentBootstrapTest.java | 8 +-
.../cassandra/fuzz/sai/SingleNodeSAITest.java | 11 +-
.../cassandra/fuzz/sai/StaticsTortureTest.java | 20 +-
.../cassandra/harry/checker/ModelChecker.java | 1 +
.../org/apache/cassandra/harry/ddl/ColumnSpec.java | 44 ++-
.../org/apache/cassandra/harry/ddl/SchemaSpec.java | 36 ++-
.../apache/cassandra/harry/dsl/ArrayWrapper.java | 49 +++
.../cassandra/harry/dsl/BatchVisitBuilder.java | 11 +-
.../apache/cassandra/harry/dsl/HistoryBuilder.java | 141 +---
.../cassandra/harry/dsl/OverridingBijection.java | 84 +
.../cassandra/harry/dsl/OverridingCkGenerator.java | 153 +
.../cassandra/harry/dsl/PartitionVisitState.java | 63 ++--
.../harry/dsl/PartitionVisitStateImpl.java | 115 +++
.../harry/dsl/ReplayingHistoryBuilder.java | 13 +-
.../harry/dsl/SingleOperationBuilder.java | 5 +-
.../harry/dsl/SingleOperationVisitBuilder.java | 72 +++--
.../harry/dsl/ValueDescriptorIndexGenerator.java | 13 +-
.../apache/cassandra/harry/dsl/ValueHelper.java| 74 +
.../apache/cassandra/harry/dsl/ValueOverrides.java | 24 ++
.../apache/cassandra/harry/gen/DataGenerators.java | 24 +-
.../apache/cassandra/harry/model/NoOpChecker.java | 16 +-
.../cassandra/harry/operations/Relation.java | 2 +-
28 files changed, 1445 insertions(+), 221 deletions(-)
diff --git
a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
index 3d3b9f3958..3e60a02523 100644
--- a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
+++ b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
@@ -545,6 +545,8 @@ public class ClusterUtils
public static void unpauseCommits(IInvokableInstance instance)
{
+if (instance.isShutdown())
+return;
instance.runOnInstance(() -> {
TestProcessor processor = (TestProcessor)
((ClusterMetadataService.SwitchableProcessor)
ClusterMetadataService.instance().processor()).delegate();
processor.unpause();
diff --git
a/test/distributed/org/apache/cassandra/fuzz/harry/examples/RepairBurnTest.java
b/test/distributed/org/apache/cassandra/fuzz/harry/examples/RepairBurnTest.java
new file mode 100644
index 00..4092d6a02f
--- /dev/null
+++
b/test/distributed/org/apache/cassandra/fuzz/harry/examples/RepairBurnTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.fuzz.harry.examples;
+
+import java.util.Arrays;
+import java.util.Random;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.fuzz.harry.integration.model.IntegrationTestBase;
+import org.apache.cassandra.harry.checker.ModelChecker;
+import org.apache.cassandra.harry.ddl.ColumnSpec;
+import org.apache.cassandra.harry.
(cassandra) branch trunk updated: TCM: Catch up committing node on rejection
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3e6a551dba TCM: Catch up committing node on rejection
3e6a551dba is described below
commit 3e6a551dbab6ecdc97b99f9ec3118316bfaf1802
Author: Alex Petrov
AuthorDate: Thu Jan 11 14:18:46 2024 +0100
TCM: Catch up committing node on rejection
Patch by Alex Petrov; reviewed by Marcus Eriksson for CASSANDRA-19260
---
.../cassandra/tcm/AbstractLocalProcessor.java | 10 +++--
src/java/org/apache/cassandra/tcm/Commit.java | 47 +++---
.../org/apache/cassandra/tcm/RemoteProcessor.java | 14 +--
...ationSmokeTest.java => LogReplicationTest.java} | 38 -
4 files changed, 94 insertions(+), 15 deletions(-)
diff --git a/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java
b/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java
index 2d00085f3d..a72b5b664f 100644
--- a/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java
+++ b/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.tcm;
+import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import org.slf4j.Logger;
@@ -79,7 +80,7 @@ public abstract class AbstractLocalProcessor implements
Processor
{
return maybeFailure(entryId,
lastKnown,
-() -> new
Commit.Result.Failure(result.rejected().code, result.rejected().reason, true));
+() ->
Commit.Result.rejected(result.rejected().code, result.rejected().reason,
toLogState(lastKnown)));
}
continue;
@@ -118,9 +119,10 @@ public abstract class AbstractLocalProcessor implements
Processor
retryPolicy.maybeSleep();
}
}
-return new Commit.Result.Failure(SERVER_ERROR,
- String.format("Could not perform
commit using the following retry stategy: %s", retryPolicy.tries),
- false);
+return Commit.Result.failed(SERVER_ERROR,
+String.format("Could not perform commit
after %d/%d tries. Time remaining: %dms",
+ retryPolicy.tries,
retryPolicy.maxTries,
+
TimeUnit.NANOSECONDS.toMillis(retryPolicy.remainingNanos(;
}
public Commit.Result maybeFailure(Entry.Id entryId, Epoch lastKnown,
Supplier orElse)
diff --git a/src/java/org/apache/cassandra/tcm/Commit.java
b/src/java/org/apache/cassandra/tcm/Commit.java
index f0146847f1..8871efa5b2 100644
--- a/src/java/org/apache/cassandra/tcm/Commit.java
+++ b/src/java/org/apache/cassandra/tcm/Commit.java
@@ -119,6 +119,9 @@ public class Commit
static volatile Result.Serializer resultSerializerCache;
public interface Result
{
+IVersionedSerializer defaultMessageSerializer = new
Serializer(NodeVersion.CURRENT.serializationVersion());
+
+LogState logState();
boolean isSuccess();
boolean isFailure();
@@ -131,7 +134,6 @@ public class Commit
{
return (Failure) this;
}
-IVersionedSerializer defaultMessageSerializer = new
Serializer(NodeVersion.CURRENT.serializationVersion());
static IVersionedSerializer messageSerializer(Version version)
{
@@ -163,6 +165,12 @@ public class Commit
'}';
}
+@Override
+public LogState logState()
+{
+return logState;
+}
+
public boolean isSuccess()
{
return true;
@@ -174,6 +182,16 @@ public class Commit
}
}
+static Failure rejected(ExceptionCode exceptionCode, String reason,
LogState logState)
+{
+return new Failure(exceptionCode, reason, logState, true);
+}
+
+static Failure failed(ExceptionCode exceptionCode, String message)
+{
+return new Failure(exceptionCode, message, LogState.EMPTY, false);
+}
+
final class Failure implements Result
{
public final ExceptionCode code;
@@ -181,8 +199,9 @@ public class Commit
// Rejection means that we were able to linearize the operation,
// but it was rejected by the internal logic of the transformation.
public final boolean rejected;
+public final LogState logState;
-public Failure(ExceptionCode code, String message, boolean
rejected)
+private Fai
(cassandra) branch trunk updated (2924762fa1 -> 061cb88b3c)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git from 2924762fa1 LocalLog: Cancel signal when unused add 061cb88b3c Improve logging for retries in TCM No new revisions were added by this update. Summary of changes: .../org/apache/cassandra/tcm/AbstractLocalProcessor.java | 7 ++- src/java/org/apache/cassandra/tcm/Retry.java | 15 +++ 2 files changed, 17 insertions(+), 5 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch trunk updated: LocalLog: Cancel signal when unused
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2924762fa1 LocalLog: Cancel signal when unused
2924762fa1 is described below
commit 2924762fa1c29c3658507225e9e5e00853ec58e6
Author: Alex Petrov
AuthorDate: Tue Jan 30 11:29:09 2024 +0100
LocalLog: Cancel signal when unused
Patch by Alex Petrov; reviewed by Marcus Eriksson and Sam Tunnicliffe for
CASSANDRA-19353.
---
src/java/org/apache/cassandra/tcm/log/LocalLog.java | 12 +++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/src/java/org/apache/cassandra/tcm/log/LocalLog.java
b/src/java/org/apache/cassandra/tcm/log/LocalLog.java
index bb878f5b12..cf7e383877 100644
--- a/src/java/org/apache/cassandra/tcm/log/LocalLog.java
+++ b/src/java/org/apache/cassandra/tcm/log/LocalLog.java
@@ -750,20 +750,24 @@ public abstract class LocalLog implements Closeable
public void run(Interruptible.State state) throws
InterruptedException
{
+WaitQueue.Signal signal = null;
try
{
if (state != Interruptible.State.SHUTTING_DOWN)
{
Condition condition = subscriber.getAndSet(null);
// Grab a ticket ahead of time, so that we can't get
into race with the exit from process pending
-WaitQueue.Signal signal = logNotifier.register();
+signal = logNotifier.register();
processPendingInternal();
if (condition != null)
condition.signalAll();
// if no new threads have subscribed since we started
running, await
// otherwise, run again to process whatever work they
may be waiting on
if (subscriber.get() == null)
+{
signal.await();
+signal = null;
+}
}
}
catch (StopProcessingException t)
@@ -780,6 +784,12 @@ public abstract class LocalLog implements Closeable
// TODO handle properly
logger.warn("Error in log follower", t);
}
+finally
+{
+// If signal was not consumed for some reason, cancel it
+if (signal != null)
+signal.cancel();
+}
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch trunk updated: Handle MIN_TOKEN placement correctly.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1cb6d3568b Handle MIN_TOKEN placement correctly.
1cb6d3568b is described below
commit 1cb6d3568b0ab9c7ea5a90b56e680d5b79c1d216
Author: Alex Petrov
AuthorDate: Mon Jan 8 17:09:27 2024 +0100
Handle MIN_TOKEN placement correctly.
Patch by Alex Petrov, reviewed by Sam Tunnicliffe for CASSANDRA-19262.
---
.../apache/cassandra/locator/SimpleStrategy.java | 9 +-
.../tcm/compatibility/TokenRingUtils.java | 5 -
.../cassandra/tcm/sequences/LockedRanges.java | 6 +
.../test/log/MetadataChangeSimulationTest.java | 152 ++---
.../distributed/test/log/ModelChecker.java | 6 +-
.../distributed/test/log/PlacementSimulator.java | 38 ++
.../test/log/PlacementSimulatorTest.java | 29 +++-
.../distributed/test/log/SimulatedOperation.java | 1 +
.../cassandra/harry/sut/TokenPlacementModel.java | 124 +++--
.../simulator/cluster/KeyspaceActions.java | 4 +-
10 files changed, 269 insertions(+), 105 deletions(-)
diff --git a/src/java/org/apache/cassandra/locator/SimpleStrategy.java
b/src/java/org/apache/cassandra/locator/SimpleStrategy.java
index 1e15eab9be..24893e5f1e 100644
--- a/src/java/org/apache/cassandra/locator/SimpleStrategy.java
+++ b/src/java/org/apache/cassandra/locator/SimpleStrategy.java
@@ -17,7 +17,11 @@
*/
package org.apache.cassandra.locator;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,9 +40,9 @@ import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.compatibility.TokenRingUtils;
import org.apache.cassandra.tcm.membership.Directory;
import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.tcm.ownership.TokenMap;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.ownership.PlacementForRange;
+import org.apache.cassandra.tcm.ownership.TokenMap;
import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
/**
@@ -159,6 +163,7 @@ public class SimpleStrategy extends
AbstractReplicationStrategy
return Collections.singleton(REPLICATION_FACTOR);
}
+@SuppressWarnings("unused") // used via reflection
protected static void prepareOptions(Map options,
Map previousOptions)
{
// When altering from NTS to SS, previousOptions could have multiple
different RFs for different data centers - so we
diff --git
a/src/java/org/apache/cassandra/tcm/compatibility/TokenRingUtils.java
b/src/java/org/apache/cassandra/tcm/compatibility/TokenRingUtils.java
index 3f4802a555..4a15925bb9 100644
--- a/src/java/org/apache/cassandra/tcm/compatibility/TokenRingUtils.java
+++ b/src/java/org/apache/cassandra/tcm/compatibility/TokenRingUtils.java
@@ -63,11 +63,6 @@ public class TokenRingUtils
return i;
}
-public static Token firstToken(List ring, Token start)
-{
-return ring.get(firstTokenIndex(ring, start, false));
-}
-
public static Token getPredecessor(List ring, Token start)
{
int idx = firstTokenIndex(ring, start, false);
diff --git a/src/java/org/apache/cassandra/tcm/sequences/LockedRanges.java
b/src/java/org/apache/cassandra/tcm/sequences/LockedRanges.java
index c4890e64ee..8693486edb 100644
--- a/src/java/org/apache/cassandra/tcm/sequences/LockedRanges.java
+++ b/src/java/org/apache/cassandra/tcm/sequences/LockedRanges.java
@@ -338,6 +338,12 @@ public class LockedRanges implements
MetadataValue
{
if (thisRange.intersects(otherRange))
return true;
+
+// Since we allow ownership of the MIN_TOKEN, we need
to lock both sides of the
+// wraparound range in case it transitions from
non-wraparound to wraparound and back.
+if ((thisRange.left.isMinimum() ||
thisRange.right.isMinimum()) &&
+(otherRange.left.isMinimum() ||
otherRange.right.isMinimum()))
+return true;
}
}
}
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
index 036e5fff71..dabe363ca0 100644
---
a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
+++
b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
@@ -45
(cassandra) branch trunk updated: Change IP address of the CMS node during transition
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 46b90364da Change IP address of the CMS node during transition
46b90364da is described below
commit 46b90364daecf1880db5eda9899d7353ad81f445
Author: Alex Petrov
AuthorDate: Thu Dec 21 13:47:22 2023 +0100
Change IP address of the CMS node during transition
Patch by Alex Petrov; reviewed by Sam Tunnicliffe and Marcus Eriksson for
CASSANDRA-19219
---
.../cassandra/locator/CMSPlacementStrategy.java| 4 --
.../cassandra/tcm/transformations/Startup.java | 20 +++
.../distributed/test/cms/CMSAddressChangeTest.java | 67 ++
.../test/log/ClusterMetadataTestHelper.java| 13 +
4 files changed, 100 insertions(+), 4 deletions(-)
diff --git a/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java
b/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java
index ea4f0cbb9a..754687a199 100644
--- a/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java
+++ b/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java
@@ -28,7 +28,6 @@ import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Transformation;
@@ -137,9 +136,6 @@ public interface CMSPlacementStrategy
public Boolean apply(ClusterMetadata metadata, NodeId nodeId)
{
-if
(!FailureDetector.instance.isAlive(metadata.directory.endpoint(nodeId)))
-return false;
-
if (metadata.directory.peerState(nodeId) != NodeState.JOINED)
return false;
diff --git a/src/java/org/apache/cassandra/tcm/transformations/Startup.java
b/src/java/org/apache/cassandra/tcm/transformations/Startup.java
index b26cc3655c..b4d4007e43 100644
--- a/src/java/org/apache/cassandra/tcm/transformations/Startup.java
+++ b/src/java/org/apache/cassandra/tcm/transformations/Startup.java
@@ -24,7 +24,10 @@ import java.util.Objects;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Transformation;
@@ -32,12 +35,14 @@ import org.apache.cassandra.tcm.membership.Directory;
import org.apache.cassandra.tcm.membership.NodeAddresses;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeVersion;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.ownership.DataPlacements;
import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.serialization.MetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
+import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange;
public class Startup implements Transformation
{
@@ -87,6 +92,21 @@ public class Startup implements Transformation
next.build().metadata,
allKeyspaces);
+if (prev.isCMSMember(prev.directory.endpoint(nodeId)))
+{
+ReplicationParams metaParams = ReplicationParams.meta(prev);
+InetAddressAndPort endpoint = prev.directory.endpoint(nodeId);
+Replica leavingReplica = new Replica(endpoint, entireRange,
true);
+Replica joiningReplica = new
Replica(addresses.broadcastAddress, entireRange, true);
+
+DataPlacement.Builder builder =
prev.placements.get(metaParams).unbuild();
+builder.reads.withoutReplica(prev.nextEpoch(), leavingReplica);
+builder.writes.withoutReplica(prev.nextEpoch(),
leavingReplica);
+builder.reads.withReplica(prev.nextEpoch(), joiningReplica);
+builder.writes.withReplica(prev.nextEpoch(), joiningReplica);
+newPlacement = newPlacement.unbuild().with(metaParams,
builder.build()).build();
+}
+
next = next.with(newPlacement);
}
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/cms/CMSAddressChangeTest.java
b/test/distributed/org/apache/cassandra/distributed/test/cms
(cassandra) branch trunk updated: Fix Harry Upgrade Test - primodal epoch initialization
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new b10e269344 Fix Harry Upgrade Test - primodal epoch initialization
b10e269344 is described below
commit b10e2693443bb5eb5c9b3d561f8d5e47ac092a8c
Author: Alex Petrov
AuthorDate: Sat Dec 23 13:30:22 2023 +0100
Fix Harry Upgrade Test - primodal epoch initialization
Patch by Alex Petrov, reviewed by Sam Tunnicliffe for CASSANDRA-19208.
---
.../cassandra/schema/DistributedMetadataLogKeyspace.java| 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git
a/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java
b/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java
index 9d16360c18..84fd433a70 100644
--- a/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java
+++ b/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java
@@ -93,7 +93,18 @@ public final class DistributedMetadataLogKeyspace
Period.FIRST,
FIRST.getEpoch(), FIRST.getEpoch(),
Transformation.Kind.PRE_INITIALIZE_CMS.toVersionedBytes(PreInitialize.blank()),
Transformation.Kind.PRE_INITIALIZE_CMS.toString(), Entry.Id.NONE.entryId);
-return result.one().getBoolean("[applied]");
+UntypedResultSet.Row row = result.one();
+if (row.getBoolean("[applied]"))
+return true;
+
+if (row.getLong("epoch") == FIRST.getEpoch() &&
+row.getLong("period") == Period.FIRST &&
+row.getLong("current_epoch") == FIRST.getEpoch() &&
+row.getLong("entry_id") == Entry.Id.NONE.entryId &&
+
Transformation.Kind.PRE_INITIALIZE_CMS.toString().equals(row.getString("kind")))
+return true;
+
+throw new IllegalStateException("Could not initialize log.");
}
catch (CasWriteTimeoutException t)
{
-
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra-harry) branch trunk updated: Fix version in run-external.sh
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git The following commit(s) were added to refs/heads/trunk by this push: new d216a71 Fix version in run-external.sh d216a71 is described below commit d216a71049ab52f522e5ed7c64423bfa3f6658e8 Author: Jaydeepkumar Chovatia AuthorDate: Tue Jan 16 22:24:12 2024 -0800 Fix version in run-external.sh --- run-external.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-external.sh b/run-external.sh index 6263110..d6a2efc 100755 --- a/run-external.sh +++ b/run-external.sh @@ -1,3 +1,3 @@ #!/bin/sh -java -Dlogback.configurationFile=test/conf/logback-dtest.xml -jar harry-integration-external/target/harry-integration-external-0.0.1-SNAPSHOT.jar conf/external.yaml +java -Dlogback.configurationFile=test/conf/logback-dtest.xml -jar harry-integration-external/target/harry-integration-external-0.0.2-SNAPSHOT.jar conf/external.yaml - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cep-15-accord updated (18ce167359 -> 1271f5f778)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git discard 18ce167359 Quick fix for AccordAddTableTest: make rejections catch up add 1271f5f778 Quick fix for AccordAddTableTest: make rejections catch up This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (18ce167359) \ N -- N -- N refs/heads/cep-15-accord (1271f5f778) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: src/java/org/apache/cassandra/service/accord/IAccordService.java | 1 - src/java/org/apache/cassandra/tcm/RemoteProcessor.java | 6 +- 2 files changed, 5 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch cep-15-accord updated (eb1572e9f5 -> 18ce167359)
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a change to branch cep-15-accord in repository https://gitbox.apache.org/repos/asf/cassandra.git from eb1572e9f5 (Accord) NPE while trying to serialize FoundKnownMap as value is null half the time but unexpected while serializing add 18ce167359 Quick fix for AccordAddTableTest: make rejections catch up No new revisions were added by this update. Summary of changes: src/java/org/apache/cassandra/service/accord/IAccordService.java | 1 + 1 file changed, 1 insertion(+) - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) branch trunk updated: Ninja: fix checkstyle after cherry-pick.
This is an automated email from the ASF dual-hosted git repository. ifesdjeen pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git The following commit(s) were added to refs/heads/trunk by this push: new bc0c8f6317 Ninja: fix checkstyle after cherry-pick. bc0c8f6317 is described below commit bc0c8f6317c71ac24c25a6bf7cf008ad1305868f Author: Alex Petrov AuthorDate: Wed Dec 20 15:08:12 2023 +0100 Ninja: fix checkstyle after cherry-pick. --- .../apache/cassandra/simulator/AlwaysDeliverNetworkScheduler.java| 5 - 1 file changed, 5 deletions(-) diff --git a/test/simulator/main/org/apache/cassandra/simulator/AlwaysDeliverNetworkScheduler.java b/test/simulator/main/org/apache/cassandra/simulator/AlwaysDeliverNetworkScheduler.java index 601f308194..f5721f5df6 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/AlwaysDeliverNetworkScheduler.java +++ b/test/simulator/main/org/apache/cassandra/simulator/AlwaysDeliverNetworkScheduler.java @@ -20,12 +20,7 @@ package org.apache.cassandra.simulator; import java.util.concurrent.TimeUnit; -import org.apache.cassandra.simulator.FutureActionScheduler; import org.apache.cassandra.simulator.systems.SimulatedTime; -import org.apache.cassandra.simulator.utils.LongRange; - -import static java.util.concurrent.TimeUnit.MICROSECONDS; -import static java.util.concurrent.TimeUnit.NANOSECONDS; /** * Action scheduler that simulates ideal networking conditions. Useful to rule out - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
(cassandra) 03/03: Remove conf/harry-example.yaml
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit b7e5675edd06f9d6bb8f22aef41c5c2caa698f7b
Author: Alex Petrov
AuthorDate: Fri Dec 8 08:26:17 2023 +0100
Remove conf/harry-example.yaml
Patch by Alex Petrov, reviewed by Sam Tunnicliffe for
CASSANDRA-19081/CASSANDRA-19208
---
conf/harry-example.yaml| 95 --
.../upgrade/ClusterMetadataUpgradeHarryTest.java | 22 +++--
2 files changed, 13 insertions(+), 104 deletions(-)
diff --git a/conf/harry-example.yaml b/conf/harry-example.yaml
deleted file mode 100644
index 1868ad9959..00
--- a/conf/harry-example.yaml
+++ /dev/null
@@ -1,95 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-seed: 1596731732524
-
-# Default schema provider generates random schema
-schema_provider:
- fixed:
-keyspace: harry
-table: test_table
-partition_keys:
- pk1: bigint
- pk2: ascii
-clustering_keys:
- ck1: ascii
- ck2: bigint
-regular_columns:
- v1: ascii
- v2: bigint
- v3: ascii
- v4: bigint
-static_keys:
- s1: ascii
- s2: bigint
- s3: ascii
- s4: bigint
-
-# Clock is a component responsible for mapping _logical_ timestamps to
_real-time_ ones.
-#
-# When reproducing test failures, and for validation purposes, a snapshot of
such clock can
-# be taken to map a real-time timestamp from the value retrieved from the
database in order
-# to map it back to the logical timestamp of the operation that wrote this
value.
-clock:
- offset:
-offset: 1000
-
-drop_schema: false
-create_schema: true
-truncate_table: true
-
-# Partition descriptor selector controls how partitions is selected based on
the current logical
-# timestamp. Default implementation is a sliding window of partition
descriptors that will visit
-# one partition after the other in the window `slide_after_repeats` times.
After that will
-# retire one partition descriptor, and pick one instead of it.
-partition_descriptor_selector:
- default:
-window_size: 10
-slide_after_repeats: 100
-
-# Clustering descriptor selector controls how clusterings are picked within
the partition:
-# how many rows there can be in a partition, how many rows will be visited for
a logical timestamp,
-# how many operations there will be in batch, what kind of operations there
will and how often
-# each kind of operation is going to occur.
-clustering_descriptor_selector:
- default:
-modifications_per_lts:
- type: "constant"
- constant: 2
-rows_per_modification:
- type: "constant"
- constant: 2
-operation_kind_weights:
- DELETE_RANGE: 0
- DELETE_SLICE: 0
- DELETE_ROW: 0
- DELETE_COLUMN: 0
- DELETE_PARTITION: 0
- DELETE_COLUMN_WITH_STATICS: 0
- INSERT_WITH_STATICS: 50
- INSERT: 50
- UPDATE_WITH_STATICS: 50
- UPDATE: 50
-column_mask_bitsets: null
-max_partition_size: 1000
-
-metric_reporter:
- no_op: {}
-
-data_tracker:
- locking:
-max_seen_lts: -1
-max_complete_lts: -1
diff --git
a/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeHarryTest.java
b/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeHarryTest.java
index ab6955bcc7..776e7b1073 100644
---
a/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeHarryTest.java
+++
b/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeHarryTest.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.distributed.upgrade;
+import java.util.Collections;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@@ -34,6 +35,8 @@ import harry.visitors.QueryLogger;
import harry.visitors.RandomPartitionValidator;
import org.apache.cassandra.distributed.Constants;
import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.fuzz.FixedSchemaProviderConfiguration;
+import org.apache.cassand
(cassandra) 01/03: Fix HarrySimulatorTest.harryTest
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 0989a219ad1242b9591d2323e9b31a79746503f8
Author: Alex Petrov
AuthorDate: Mon Dec 4 17:09:38 2023 +0100
Fix HarrySimulatorTest.harryTest
Patch by Alex Petrov, reviewed by Sam Tunnicliffe for CASSANDRA-19094.
---
.../cassandra/tcm/sequences/ProgressBarrier.java | 16 +-
.../simulator/asm/GlobalMethodTransformer.java | 3 +-
.../cassandra/simulator/ClusterSimulation.java | 45 ++-
.../cassandra/simulator/SimulationRunner.java | 4 +-
.../apache/cassandra/simulator/SimulatorUtils.java | 9 -
.../simulator/cluster/KeyspaceActions.java | 5 +-
.../simulator/cluster/OnClusterLeave.java | 3 +-
.../simulator/cluster/OnClusterReplace.java| 3 +-
.../cassandra/simulator/debug/Reconcile.java | 4 +-
.../apache/cassandra/simulator/debug/Record.java | 41 ++-
.../cassandra/simulator/debug/SelfReconcile.java | 15 +-
.../simulator/harry/HarryValidatingQuery.java | 3 +-
.../simulator/test/HarrySimulatorTest.java | 384 +++--
.../simulator/test/ShortPaxosSimulationTest.java | 4 +-
.../simulator/test/SimulationTestBase.java | 4 +-
15 files changed, 371 insertions(+), 172 deletions(-)
diff --git a/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java
b/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java
index 3bc5db2782..28f94cacf8 100644
--- a/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java
+++ b/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java
@@ -194,6 +194,7 @@ public class ProgressBarrier
for (InetAddressAndPort peer : superset)
requests.add(new WatermarkRequest(peer, messagingService,
waitFor));
+long start = Clock.Global.nanoTime();
Retry.Deadline deadline =
Retry.Deadline.after(TimeUnit.MILLISECONDS.toNanos(TIMEOUT_MILLIS),
new
Retry.Backoff(DatabaseDescriptor.getCmsDefaultRetryMaxTries(),
(int)
BACKOFF_MILLIS,
@@ -241,8 +242,8 @@ public class ProgressBarrier
Set remaining = new HashSet<>(superset);
remaining.removeAll(collected);
-logger.warn("Could not collect {} of nodes for a progress barrier for
epoch {} to finish within {}ms. Nodes that have not responded: {}",
-cl, waitFor,
TimeUnit.NANOSECONDS.toMillis(Clock.Global.nanoTime() -
deadline.deadlineNanos), remaining);
+logger.warn("Could not collect {} of nodes for a progress barrier for
epoch {} to finish within {}ms. Nodes that have not responded: {}. {}",
+cl, waitFor,
TimeUnit.NANOSECONDS.toMillis(deadline.deadlineNanos - start), remaining,
deadline);
return false;
}
@@ -555,6 +556,17 @@ public class ProgressBarrier
condition = new AsyncPromise<>();
messagingService.sendWithCallback(Message.out(Verb.TCM_CURRENT_EPOCH_REQ,
ClusterMetadata.current().epoch), to, this);
}
+
+@Override
+public String toString()
+{
+return "WatermarkRequest{" +
+ "condition=" + condition +
+ ", to=" + to +
+ ", messagingService=" + messagingService +
+ ", waitFor=" + waitFor +
+ '}';
+}
}
@Override
diff --git
a/test/simulator/asm/org/apache/cassandra/simulator/asm/GlobalMethodTransformer.java
b/test/simulator/asm/org/apache/cassandra/simulator/asm/GlobalMethodTransformer.java
index 67650639e0..883b7a66a3 100644
---
a/test/simulator/asm/org/apache/cassandra/simulator/asm/GlobalMethodTransformer.java
+++
b/test/simulator/asm/org/apache/cassandra/simulator/asm/GlobalMethodTransformer.java
@@ -92,7 +92,8 @@ class GlobalMethodTransformer extends MethodVisitor
super.visitMethodInsn(Opcodes.INVOKESTATIC,
"org/apache/cassandra/simulator/systems/InterceptorOfSystemMethods$Global",
name, descriptor, false);
}
else if ((globalMethods || deterministic) && opcode ==
Opcodes.INVOKESTATIC &&
-owner.equals("java/util/concurrent/ThreadLocalRandom") &&
(name.equals("getProbe") || name.equals("advanceProbe") ||
name.equals("localInit"))
+ ((owner.equals("java/util/concurrent/ThreadLocalRandom") &&
(name.equals("getProbe") || name.equals("advanceProbe") ||
name.equals("localInit")))
+ || (owner.equals("java/util/concurrent/atomic/Striped64") &&
(name.equals("
