(ignite) branch master updated: IGNITE-14823 Abbrevation checkstyle rule (#9153)
This is an automated email from the ASF dual-hosted git repository. nizhikov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new 6ccc32e2cbe IGNITE-14823 Abbrevation checkstyle rule (#9153) 6ccc32e2cbe is described below commit 6ccc32e2cbeda36026a8ebffa1fd4b04839bfb9e Author: Nikolay AuthorDate: Sat Dec 9 09:05:07 2023 +0300 IGNITE-14823 Abbrevation checkstyle rule (#9153) --- checkstyle/checkstyle.xml | 3 + modules/checkstyle/pom.xml | 16 ++ .../tools/checkstyle/IgniteAbbrevationsRule.java | 177 + .../src/main/resources/abbrevations-exclude.txt| 1 + .../checkstyle/src/main/resources/abbrevations.csv | 71 + .../checkstyle/IgniteAbbrevationRuleTest.java | 95 +++ .../apache/ignite/util/GridCommandHandlerTest.java | 8 +- .../maintenance/MaintenanceRebuildIndexUtils.java | 14 +- .../jdbc/thin/ConnectionPropertiesImpl.java| 14 +- .../management/baseline/BaselineTaskResult.java| 4 +- .../systemview/IgniteConfigurationIterable.java| 6 +- .../wal/filehandle/FileWriteHandleImpl.java| 6 +- .../apache/ignite/internal/util/IgniteUtils.java | 4 +- .../ignite/internal/util/io/GridFilenameUtils.java | 56 +++ .../sharedfs/TcpDiscoverySharedFsIpFinder.java | 4 +- .../tcp/ipfinder/vm/TcpDiscoveryVmIpFinder.java| 10 +- ...rNodeAttributeAffinityBackupFilterSelfTest.java | 6 +- .../internal/client/thin/ReliableChannelTest.java | 4 +- .../optimized/OptimizedObjectStreamSelfTest.java | 4 +- .../GridCacheBinaryObjectsAbstractSelfTest.java| 4 +- .../cache/persistence/file/FileDownloaderTest.java | 6 +- .../persistence/pagemem/PageMemoryImplTest.java| 8 +- .../ignite/internal/util/IgniteUtilsSelfTest.java | 16 +- .../apache/ignite/util/GridLongListSelfTest.java | 14 +- .../utils/IgniteWalConverterArguments.java | 4 +- .../persistence/db/wal/IgniteWalRecoveryTest.java | 10 +- .../query/WrongQueryEntityFieldTypeTest.java | 4 +- ...achePartitionedAtomicColumnConstraintsTest.java | 8 +- .../org/apache/ignite/sqltests/BaseSqlTest.java| 4 +- .../KubernetesServiceAddressResolver.java | 4 +- parent/pom.xml | 9 +- 31 files changed, 482 insertions(+), 112 deletions(-) diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml index 513016577da..b64eba3540d 100644 --- a/checkstyle/checkstyle.xml +++ b/checkstyle/checkstyle.xml @@ -191,6 +191,9 @@ + + +
(ignite-3) branch main updated: IGNITE-20874 Node cleanup procedure (#2877)
This is an automated email from the ASF dual-hosted git repository.
vpyatkov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 5d6b90d309 IGNITE-20874 Node cleanup procedure (#2877)
5d6b90d309 is described below
commit 5d6b90d309bbc7f79ff7f43863507c8d8c3042de
Author: Cyrill
AuthorDate: Fri Dec 8 20:24:13 2023 +0300
IGNITE-20874 Node cleanup procedure (#2877)
---
.../apache/ignite/client/fakes/FakeTxManager.java | 10 +-
.../ignite/internal/table/ItDurableFinishTest.java | 4 +-
.../ItTxDistributedCleanupRecoveryTest.java| 144 ++-
...xDistributedTestSingleNodeNoCleanupMessage.java | 6 +-
.../table/distributed/StorageUpdateHandler.java| 12 +-
.../table/distributed/TableMessageGroup.java | 6 +-
...pCommand.java => WriteIntentSwitchCommand.java} | 4 +-
.../table/distributed/raft/PartitionListener.java | 12 +-
.../replicator/PartitionReplicaListener.java | 145 +--
.../table/distributed/IndexCleanupTest.java| 12 +-
.../table/distributed/StorageCleanupTest.java | 38 ++---
.../PartitionRaftCommandsSerializationTest.java| 20 +--
.../raft/PartitionCommandListenerTest.java | 20 +--
.../PartitionReplicaListenerDurableUnlockTest.java | 47 ++
.../replication/PartitionReplicaListenerTest.java | 16 +--
.../table/impl/DummyInternalTableImpl.java | 65 -
.../org/apache/ignite/internal/tx/TxManager.java | 17 ++-
.../internal/tx/impl/PlacementDriverHelper.java| 148 +++
.../internal/tx/impl/TxCleanupRequestHandler.java | 158 +
.../internal/tx/impl/TxCleanupRequestSender.java | 151
.../ignite/internal/tx/impl/TxManagerImpl.java | 114 ++-
.../ignite/internal/tx/impl/TxMessageSender.java | 158 +
.../tx/impl/WriteIntentSwitchProcessor.java| 121
...upReplicaRequest.java => TxCleanupMessage.java} | 27 ++--
.../tx/message/TxCleanupMessageErrorResponse.java | 35 +
.../tx/message/TxCleanupMessageResponse.java | 28
.../ignite/internal/tx/message/TxMessageGroup.java | 19 ++-
...t.java => WriteIntentSwitchReplicaRequest.java} | 12 +-
modules/transactions/tech-notes/cleanup.puml | 51 +++
29 files changed, 1147 insertions(+), 453 deletions(-)
diff --git
a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeTxManager.java
b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeTxManager.java
index 43ff6feaa9..dce43dd8d2 100644
---
a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeTxManager.java
+++
b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeTxManager.java
@@ -19,6 +19,7 @@ package org.apache.ignite.client.fakes;
import static
org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
+import java.util.Collection;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@@ -176,16 +177,15 @@ public class FakeTxManager implements TxManager {
Map enlistedGroups,
UUID txId
) {
-return null;
+return nullCompletedFuture();
}
@Override
public CompletableFuture cleanup(
-String primaryConsistentId,
-TablePartitionId tablePartitionId,
-UUID txId,
+Collection partitions,
boolean commit,
-@Nullable HybridTimestamp commitTimestamp
+@Nullable HybridTimestamp commitTimestamp,
+UUID txId
) {
return nullCompletedFuture();
}
diff --git
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItDurableFinishTest.java
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItDurableFinishTest.java
index c55f46f26e..ab22a7f004 100644
---
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItDurableFinishTest.java
+++
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItDurableFinishTest.java
@@ -40,8 +40,8 @@ import org.apache.ignite.internal.replicator.TablePartitionId;
import org.apache.ignite.internal.testframework.IgniteTestUtils;
import org.apache.ignite.internal.tx.InternalTransaction;
import org.apache.ignite.internal.tx.TxMeta;
-import org.apache.ignite.internal.tx.message.TxCleanupReplicaRequest;
import org.apache.ignite.internal.tx.message.TxFinishReplicaRequest;
+import org.apache.ignite.internal.tx.message.WriteIntentSwitchReplicaRequest;
import org.apache.ignite.internal.tx.storage.state.TxStateStorage;
import org.apache.ignite.network.ClusterService;
import org.apache.ignite.network.DefaultMessagingService;
@@ -229,7 +229,7 @@ public class ItDurableFinishTest extends
ClusterPerTestIntegrationTest {
// Make sure
(ignite-3) 01/01: Hang on commit
This is an automated email from the ASF dual-hosted git repository.
ascherbakov pushed a commit to branch tx_deadlock_recovery_hang
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit e7f3ff5fa9f1f6fd3c8a39843677035a85be5bb8
Author: Alexey Scherbakov
AuthorDate: Fri Dec 8 18:40:59 2023 +0300
Hang on commit
---
.../apache/ignite/distributed/ItLockTableTest.java | 203 +
1 file changed, 203 insertions(+)
diff --git
a/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItLockTableTest.java
b/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItLockTableTest.java
new file mode 100644
index 00..dfe256571b
--- /dev/null
+++
b/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItLockTableTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.distributed;
+
+import static
org.apache.ignite.internal.replicator.ReplicaManager.DEFAULT_IDLE_SAFE_TIME_PROPAGATION_PERIOD_MILLISECONDS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import
org.apache.ignite.internal.configuration.testframework.ConfigurationExtension;
+import
org.apache.ignite.internal.configuration.testframework.InjectConfiguration;
+import org.apache.ignite.internal.hlc.HybridClock;
+import org.apache.ignite.internal.logger.IgniteLogger;
+import org.apache.ignite.internal.logger.Loggers;
+import org.apache.ignite.internal.placementdriver.PlacementDriver;
+import org.apache.ignite.internal.raft.configuration.RaftConfiguration;
+import org.apache.ignite.internal.replicator.ReplicaService;
+import org.apache.ignite.internal.schema.Column;
+import org.apache.ignite.internal.schema.SchemaDescriptor;
+import org.apache.ignite.internal.schema.configuration.GcConfiguration;
+import org.apache.ignite.internal.table.TableViewInternal;
+import org.apache.ignite.internal.testframework.IgniteAbstractTest;
+import org.apache.ignite.internal.tx.DeadlockPreventionPolicy;
+import org.apache.ignite.internal.tx.HybridTimestampTracker;
+import org.apache.ignite.internal.tx.InternalTransaction;
+import org.apache.ignite.internal.tx.configuration.TransactionConfiguration;
+import org.apache.ignite.internal.tx.impl.HeapLockManager;
+import org.apache.ignite.internal.tx.impl.TransactionIdGenerator;
+import org.apache.ignite.internal.tx.impl.TxManagerImpl;
+import org.apache.ignite.internal.type.NativeTypes;
+import org.apache.ignite.network.ClusterNode;
+import org.apache.ignite.network.ClusterService;
+import org.apache.ignite.table.RecordView;
+import org.apache.ignite.table.Tuple;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * Test lock table.
+ */
+@ExtendWith(ConfigurationExtension.class)
+public class ItLockTableTest extends IgniteAbstractTest {
+private static final IgniteLogger LOG =
Loggers.forClass(ItLockTableTest.class);
+
+private static int EMP_TABLE_ID = 2;
+
+private static final int CACHE_SIZE = 10;
+
+private static final String TABLE_NAME = "test";
+
+private static SchemaDescriptor TABLE_SCHEMA = new SchemaDescriptor(
+1,
+new Column[]{new Column("id".toUpperCase(), NativeTypes.INT32,
false)},
+new Column[]{
+new Column("name".toUpperCase(), NativeTypes.STRING, true),
+new Column("salary".toUpperCase(), NativeTypes.DOUBLE,
true)
+}
+);
+
+protected TableViewInternal testTable;
+
+protected final TestInfo testInfo;
+
+//TODO fsync can be turned on again after
https://issues.apache.org/jira/browse/IGNITE-20195
+@InjectConfiguration("mock: { fsync: false }")
+protected static RaftConfiguration raftConfiguration;
+
+@InjectConfiguration
+protected static GcConfiguration gcConfig;
+
+@InjectConfiguration
+protected static TransactionConfiguration txConfiguration;
+
+private ItTxTestCluster txTestCluster;
+
+private HybridTimestampTracker
(ignite-3) branch tx_deadlock_recovery_hang created (now e7f3ff5fa9)
This is an automated email from the ASF dual-hosted git repository. ascherbakov pushed a change to branch tx_deadlock_recovery_hang in repository https://gitbox.apache.org/repos/asf/ignite-3.git at e7f3ff5fa9 Hang on commit This branch includes the following new commits: new e7f3ff5fa9 Hang on commit The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(ignite) 04/05: IGNITE-21041 Make FilteredRecord local for RecordSerializer (#11082)
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a commit to branch ignite-2.16
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 369644c989e85aa4c889d89a2a9de69cf8124f05
Author: Maksim Timonin
AuthorDate: Fri Dec 8 15:23:42 2023 +0300
IGNITE-21041 Make FilteredRecord local for RecordSerializer (#11082)
---
.../apache/ignite/internal/pagemem/wal/record/FilteredRecord.java| 3 ---
.../cache/persistence/wal/reader/StandaloneWalRecordsIterator.java | 5 -
.../cache/persistence/wal/serializer/RecordV1Serializer.java | 5 -
.../cache/persistence/wal/serializer/RecordV2Serializer.java | 5 -
4 files changed, 12 insertions(+), 6 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
index e077e5accc2..519e825953d 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
@@ -24,9 +24,6 @@ import
org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRe
* {@link AbstractWalRecordsIterator}.
*/
public class FilteredRecord extends WALRecord {
-/** Instance. */
-public static final FilteredRecord INSTANCE = new FilteredRecord();
-
/** {@inheritDoc} */
@Override public RecordType type() {
return null;
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index c7582b7cf14..26dd7c3e647 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -96,6 +96,9 @@ class StandaloneWalRecordsIterator extends
AbstractWalRecordsIterator {
/** Replay from bound include. */
private final WALPointer lowBound;
+/** Singleton instance of {@link FilteredRecord} */
+private final WALRecord filteredRecord = new FilteredRecord();
+
/**
* Creates iterator in file-by-file iteration mode. Directory
*
@@ -286,7 +289,7 @@ class StandaloneWalRecordsIterator extends
AbstractWalRecordsIterator {
return tup;
if (!checkBounds(tup.get1()))
-return new T2<>(tup.get1(), FilteredRecord.INSTANCE); //
FilteredRecord for mark as filtered.
+return new T2<>(tup.get1(), filteredRecord); // FilteredRecord for
mark as filtered.
return tup;
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
index 295e174abf8..52c360d47eb 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -98,6 +98,9 @@ public class RecordV1Serializer implements RecordSerializer {
*/
private final boolean marshalledMode;
+/** Singleton instance of {@link FilteredRecord} */
+private final FilteredRecord filteredRecord = new FilteredRecord();
+
/** Thread-local heap byte buffer. */
private final ThreadLocal heapTlb = new
ThreadLocal() {
@Override protected ByteBuffer initialValue() {
@@ -148,7 +151,7 @@ public class RecordV1Serializer implements RecordSerializer
{
if (recType.purpose() != WALRecord.RecordPurpose.INTERNAL
&& recordFilter != null && !recordFilter.apply(rec.type(),
ptr))
-return FilteredRecord.INSTANCE;
+return filteredRecord;
else if (marshalledMode) {
ByteBuffer buf = heapTlb.get();
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
index 60c68c13f7c..21aa52a91ba 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
@@ -74,6 +74,9 @@ public class RecordV2Serializer implements RecordSerializer {
/** Skip position
(ignite) branch ignite-2.16 updated (d18d899ba2e -> 6b2d1773af7)
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a change to branch ignite-2.16
in repository https://gitbox.apache.org/repos/asf/ignite.git
from d18d899ba2e [MINOR] Fix error message in test
new f46fa294cb2 IGNITE-17700 Introduce CdcManager (#11044)
new 654b467d779 IGNITE-21042 Move CreateDumpFutureTask#toLong to
IgniteUtils (#11048)
new 5ab68b2be79 IGNITE-20822 Filter by higherBound in
AbstractWalRecordsIterator (#11034)
new 369644c989e IGNITE-21041 Make FilteredRecord local for
RecordSerializer (#11082)
new 6b2d1773af7 IGNITE-21046 Move binaryMetadataUpdateListener to basic
interface (#11049)
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../ignite/internal/cdc/CdcConsumerState.java | 39 ++
.../org/apache/ignite/internal/cdc/CdcMain.java| 240 +---
.../org/apache/ignite/internal/cdc/CdcManager.java | 123 ++
.../org/apache/ignite/internal/cdc/CdcMode.java} | 19 +-
.../internal/cdc/CdcUtilityActiveCdcManager.java | 73
.../{CdcDataRecord.java => CdcManagerRecord.java} | 30 +-
...dcDataRecord.java => CdcManagerStopRecord.java} | 21 +-
.../pagemem/wal/record/FilteredRecord.java | 3 -
.../internal/pagemem/wal/record/WALRecord.java | 8 +-
.../processors/cache/GridCacheProcessor.java | 27 ++
.../processors/cache/GridCacheSharedContext.java | 37 +-
.../binary/CacheObjectBinaryProcessorImpl.java | 6 +-
.../IgniteCacheDatabaseSharedManager.java | 4 +
.../snapshot/dump/CreateDumpFutureTask.java| 8 +-
.../wal/AbstractWalRecordsIterator.java| 10 +
.../persistence/wal/FileWriteAheadLogManager.java | 19 +-
.../wal/SingleSegmentLogicalRecordsIterator.java | 9 +-
.../wal/filehandle/FileWriteHandleImpl.java| 42 ++-
.../wal/reader/StandaloneWalRecordsIterator.java | 19 +-
.../wal/serializer/RecordDataV1Serializer.java | 34 ++
.../wal/serializer/RecordV1Serializer.java | 5 +-
.../wal/serializer/RecordV2Serializer.java | 5 +-
.../cacheobject/IgniteCacheObjectProcessor.java| 8 +
.../apache/ignite/internal/util/IgniteUtils.java | 11 +
.../ignite/cdc/CdcIgniteNodeActiveModeTest.java| 394 +++
.../java/org/apache/ignite/cdc/CdcManagerTest.java | 415 +
.../java/org/apache/ignite/cdc/WalForCdcTest.java | 4 +
.../snapshot/dump/AbstractCacheDumpTest.java | 2 +-
.../testframework/wal/record/RecordUtils.java | 18 +
.../ignite/testsuites/IgnitePdsTestSuite2.java | 4 +
30 files changed, 1485 insertions(+), 152 deletions(-)
create mode 100644
modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcManager.java
copy
modules/{calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/agg/AggregateType.java
=> core/src/main/java/org/apache/ignite/internal/cdc/CdcMode.java} (75%)
create mode 100644
modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcUtilityActiveCdcManager.java
copy
modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/{CdcDataRecord.java
=> CdcManagerRecord.java} (53%)
copy
modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/{CdcDataRecord.java
=> CdcManagerStopRecord.java} (66%)
create mode 100644
modules/core/src/test/java/org/apache/ignite/cdc/CdcIgniteNodeActiveModeTest.java
create mode 100644
modules/core/src/test/java/org/apache/ignite/cdc/CdcManagerTest.java
(ignite) 03/05: IGNITE-20822 Filter by higherBound in AbstractWalRecordsIterator (#11034)
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a commit to branch ignite-2.16
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 5ab68b2be79f28ed4cb72c95efb223b033412826
Author: Maksim Timonin
AuthorDate: Mon Nov 13 17:39:43 2023 +0300
IGNITE-20822 Filter by higherBound in AbstractWalRecordsIterator (#11034)
---
.../persistence/wal/AbstractWalRecordsIterator.java | 10 ++
.../cache/persistence/wal/FileWriteAheadLogManager.java | 13 +
.../wal/SingleSegmentLogicalRecordsIterator.java | 9 -
.../wal/reader/StandaloneWalRecordsIterator.java | 16 ++--
4 files changed, 25 insertions(+), 23 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index 926bce8fe39..c020a30be29 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -97,6 +97,9 @@ public abstract class AbstractWalRecordsIterator
/** Factory to provide I/O interfaces for read primitives with files. */
private final SegmentFileInputFactory segmentFileInputFactory;
+/** Optional inclusive high bound. */
+protected final @Nullable WALPointer highBound;
+
/** Position of last read valid record. */
private WALPointer lastRead;
@@ -106,6 +109,7 @@ public abstract class AbstractWalRecordsIterator
* @param serializerFactory Serializer of current version to read headers.
* @param ioFactory ioFactory for file IO access.
* @param initialReadBufferSize buffer for reading records size.
+ * @param highBound Optional inclusive high bound.
* @param segmentFileInputFactory Factory to provide I/O interfaces for
read primitives with files.
*/
protected AbstractWalRecordsIterator(
@@ -114,12 +118,14 @@ public abstract class AbstractWalRecordsIterator
@NotNull final RecordSerializerFactory serializerFactory,
@NotNull final FileIOFactory ioFactory,
final int initialReadBufferSize,
+@Nullable WALPointer highBound,
SegmentFileInputFactory segmentFileInputFactory) {
this.log = log;
this.sharedCtx = sharedCtx;
this.serializerFactory = serializerFactory;
this.ioFactory = ioFactory;
this.segmentFileInputFactory = segmentFileInputFactory;
+this.highBound = highBound;
buf = new ByteBufferExpander(initialReadBufferSize,
ByteOrder.nativeOrder());
}
@@ -269,6 +275,10 @@ public abstract class AbstractWalRecordsIterator
WALPointer actualFilePtr = new WALPointer(hnd.idx(),
(int)hnd.in().position(), 0);
+// Fast stop condition, after high bound reached.
+if (highBound != null && actualFilePtr.compareTo(highBound) > 0)
+return null;
+
try {
WALRecord rec = hnd.ser().readRecord(hnd.in(), actualFilePtr);
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 2b74c48d8c4..12c40e81081 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -2976,9 +2976,6 @@ public class FileWriteAheadLogManager extends
GridCacheSharedManagerAdapter impl
/** Optional start pointer. */
@Nullable private final WALPointer start;
-/** Optional end pointer. */
-@Nullable private final WALPointer end;
-
/** Manager of segment location. */
private final SegmentRouter segmentRouter;
@@ -3022,6 +3019,7 @@ public class FileWriteAheadLogManager extends
GridCacheSharedManagerAdapter impl
serializerFactory,
ioFactory,
dsCfg.getWalRecordIteratorBufferSize(),
+end,
segmentFileInputFactory
);
@@ -3029,7 +3027,6 @@ public class FileWriteAheadLogManager extends
GridCacheSharedManagerAdapter impl
this.walWorkDir = walWorkDir;
this.archiver = archiver;
this.start = start;
-this.end = end;
this.dsCfg = dsCfg;
this.decompressor = decompressor;
@@ -3116,7 +3113,7 @@ public class FileWriteAheadLogManager extends
GridCacheSharedManagerAdapter impl
curWalSegmIdx--;
if
(ignite) 01/05: IGNITE-17700 Introduce CdcManager (#11044)
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a commit to branch ignite-2.16
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit f46fa294cb21bdf1b979ad2d0866bdc28ed9edc6
Author: Maksim Timonin
AuthorDate: Thu Dec 7 19:33:58 2023 +0300
IGNITE-17700 Introduce CdcManager (#11044)
Co-authored-by: Nikolay Izhikov
---
.../ignite/internal/cdc/CdcConsumerState.java | 39 ++
.../org/apache/ignite/internal/cdc/CdcMain.java| 240 +---
.../org/apache/ignite/internal/cdc/CdcManager.java | 123 ++
.../org/apache/ignite/internal/cdc/CdcMode.java| 27 ++
.../internal/cdc/CdcUtilityActiveCdcManager.java | 73
.../pagemem/wal/record/CdcManagerRecord.java | 50 +++
.../pagemem/wal/record/CdcManagerStopRecord.java | 33 ++
.../internal/pagemem/wal/record/WALRecord.java | 8 +-
.../processors/cache/GridCacheProcessor.java | 27 ++
.../processors/cache/GridCacheSharedContext.java | 37 +-
.../IgniteCacheDatabaseSharedManager.java | 4 +
.../persistence/wal/FileWriteAheadLogManager.java | 6 +-
.../wal/filehandle/FileWriteHandleImpl.java| 42 ++-
.../wal/serializer/RecordDataV1Serializer.java | 34 ++
.../ignite/cdc/CdcIgniteNodeActiveModeTest.java| 394 +++
.../java/org/apache/ignite/cdc/CdcManagerTest.java | 415 +
.../java/org/apache/ignite/cdc/WalForCdcTest.java | 4 +
.../testframework/wal/record/RecordUtils.java | 18 +
.../ignite/testsuites/IgnitePdsTestSuite2.java | 4 +
19 files changed, 1501 insertions(+), 77 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcConsumerState.java
b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcConsumerState.java
index 379c2bb6f95..942fd248da6 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcConsumerState.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcConsumerState.java
@@ -71,6 +71,15 @@ public class CdcConsumerState {
/** */
public static final String CACHES_STATE_FILE_NAME = "cdc-caches-state" +
FILE_SUFFIX;
+/**
+ * The file stores state of CDC mode. Content of the file is a {@link
CdcMode} value:
+ *
+ * {@link CdcMode#CDC_UTILITY_ACTIVE} means that {@link CdcMain}
utility captures data.
+ * {@link CdcMode#IGNITE_NODE_ACTIVE} means that {@link
CdcManager} captures data within Ignite node.
+ *
+ */
+public static final String CDC_MODE_FILE_NAME = "cdc-mode" + FILE_SUFFIX;
+
/** Log. */
private final IgniteLogger log;
@@ -98,6 +107,12 @@ public class CdcConsumerState {
/** Mappings types state file. */
private final Path tmpCaches;
+/** CDC manager mode state file. */
+private final Path cdcMode;
+
+/** Temp CDC manager mode state file. */
+private final Path tmpCdcMode;
+
/**
* @param stateDir State directory.
*/
@@ -111,6 +126,8 @@ public class CdcConsumerState {
tmpMappings = stateDir.resolve(MAPPINGS_STATE_FILE_NAME + TMP_SUFFIX);
caches = stateDir.resolve(CACHES_STATE_FILE_NAME);
tmpCaches = stateDir.resolve(CACHES_STATE_FILE_NAME + TMP_SUFFIX);
+cdcMode = stateDir.resolve(CDC_MODE_FILE_NAME);
+tmpCdcMode = stateDir.resolve(CDC_MODE_FILE_NAME + TMP_SUFFIX);
}
/**
@@ -278,4 +295,26 @@ public class CdcConsumerState {
throw new RuntimeException(e);
}
}
+
+/**
+ * Loads CDC mode state from file.
+ *
+ * @return CDC mode state.
+ */
+public CdcMode loadCdcMode() {
+CdcMode state = load(cdcMode, () -> CdcMode.IGNITE_NODE_ACTIVE);
+
+log.info("CDC mode loaded [" + state + ']');
+
+return state;
+}
+
+/**
+ * Saves CDC mode state to file.
+ *
+ * @param mode CDC mode.
+ */
+public void saveCdcMode(CdcMode mode) throws IOException {
+save(mode, tmpCdcMode, cdcMode);
+}
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java
b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java
index 84dc0a02dc9..cbf44b0b11e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java
@@ -50,6 +50,9 @@ import
org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.MarshallerContextImpl;
import org.apache.ignite.internal.binary.BinaryUtils;
import org.apache.ignite.internal.cdc.WalRecordsConsumer.DataEntryIterator;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.record.CdcManagerRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.GridLocalConfigManager;
import
(ignite) 05/05: IGNITE-21046 Move binaryMetadataUpdateListener to basic interface (#11049)
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a commit to branch ignite-2.16
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 6b2d1773af715ab2eab4afe0138294dbefb3c24f
Author: Maksim Timonin
AuthorDate: Fri Dec 8 15:29:19 2023 +0300
IGNITE-21046 Move binaryMetadataUpdateListener to basic interface (#11049)
---
.../processors/cache/binary/CacheObjectBinaryProcessorImpl.java | 6 ++
.../processors/cacheobject/IgniteCacheObjectProcessor.java| 8
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 7ca52e66449..4e1bc0b59c8 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -372,10 +372,8 @@ public class CacheObjectBinaryProcessorImpl extends
GridProcessorAdapter impleme
}
}
-/**
- * @param lsnr Listener.
- */
-public void addBinaryMetadataUpdateListener(BinaryMetadataUpdatedListener
lsnr) {
+/** {@inheritDoc} */
+@Override public void
addBinaryMetadataUpdateListener(BinaryMetadataUpdatedListener lsnr) {
if (transport != null)
transport.addBinaryMetadataUpdateListener(lsnr);
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
index 68953161430..1b3cd9bcc88 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
@@ -39,6 +39,7 @@ import
org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.IncompleteCacheObject;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import
org.apache.ignite.internal.processors.cache.binary.BinaryMetadataUpdatedListener;
import org.jetbrains.annotations.Nullable;
/**
@@ -377,4 +378,11 @@ public interface IgniteCacheObjectProcessor extends
GridProcessor {
* @throws org.apache.ignite.binary.BinaryObjectException In case of error.
*/
public BinaryType registerClass(Class cls) throws BinaryObjectException;
+
+/**
+ * Add binary metadata update listener.
+ *
+ * @param lsnr Listener.
+ */
+public void addBinaryMetadataUpdateListener(BinaryMetadataUpdatedListener
lsnr);
}
(ignite) 02/05: IGNITE-21042 Move CreateDumpFutureTask#toLong to IgniteUtils (#11048)
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a commit to branch ignite-2.16
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 654b467d779550444367d5fd844bb8c2680a6a48
Author: Maksim Timonin
AuthorDate: Fri Dec 8 15:16:57 2023 +0300
IGNITE-21042 Move CreateDumpFutureTask#toLong to IgniteUtils (#11048)
---
.../cache/persistence/snapshot/dump/CreateDumpFutureTask.java | 8 ++--
.../java/org/apache/ignite/internal/util/IgniteUtils.java | 11 +++
.../persistence/snapshot/dump/AbstractCacheDumpTest.java | 2 +-
3 files changed, 14 insertions(+), 7 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
index d0654e719b0..3f181b71a97 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
@@ -75,6 +75,7 @@ import static
org.apache.ignite.internal.processors.cache.persistence.file.FileP
import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_GRP_DIR_PREFIX;
import static
org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.DUMP_LOCK;
import static
org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.Dump.dumpPartFileName;
+import static org.apache.ignite.internal.util.IgniteUtils.toLong;
/**
* Task creates cache group dump.
@@ -111,7 +112,7 @@ public class CreateDumpFutureTask extends
AbstractCreateSnapshotFutureTask imple
* Dump contextes.
* Key is [group_id, partition_id] combined in single long value.
*
- * @see #toLong(int, int)
+ * @see IgniteUtils#toLong(int, int)
*/
private final Map dumpCtxs = new
ConcurrentHashMap<>();
@@ -631,11 +632,6 @@ public class CreateDumpFutureTask extends
AbstractCreateSnapshotFutureTask imple
}
}
-/** */
-public static long toLong(int high, int low) {
-return (((long)high) << Integer.SIZE) | (low & 0xL);
-}
-
/** */
private File groupDirectory(CacheGroupContext grpCtx) throws
IgniteCheckedException {
return new File(
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 2f736091ed1..de17a4eea7f 100755
---
a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -7704,6 +7704,17 @@ public abstract class IgniteUtils {
return ret;
}
+/**
+ * Concats two integers to long.
+ *
+ * @param high Highest bits.
+ * @param low Lowest bits.
+ * @return Long.
+ */
+public static long toLong(int high, int low) {
+return (((long)high) << Integer.SIZE) | (low & 0xL);
+}
+
/**
* Copies all elements from collection to array and asserts that
* array is big enough to hold the collection. This method should
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java
index 75d9bdda06b..bdcfab73f25 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java
@@ -79,7 +79,7 @@ import static
org.apache.ignite.internal.encryption.AbstractEncryptionTest.KEYST
import static
org.apache.ignite.internal.encryption.AbstractEncryptionTest.KEYSTORE_PATH;
import static
org.apache.ignite.internal.processors.cache.GridCacheUtils.UTILITY_CACHE_NAME;
import static
org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.SNP_RUNNING_DIR_KEY;
-import static
org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.CreateDumpFutureTask.toLong;
+import static org.apache.ignite.internal.util.IgniteUtils.toLong;
import static org.apache.ignite.platform.model.AccessLevel.SUPER;
import static org.apache.ignite.testframework.GridTestUtils.runAsync;
import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
(ignite-3) branch main updated: IGNITE-21024 Public methods of transaction manager might throw NodeStoppingException (#2938)
This is an automated email from the ASF dual-hosted git repository.
vpyatkov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 25411b947c IGNITE-21024 Public methods of transaction manager might
throw NodeStoppingException (#2938)
25411b947c is described below
commit 25411b947c9089f07c52c907f79e7823d020ea29
Author: Cyrill
AuthorDate: Fri Dec 8 15:52:04 2023 +0300
IGNITE-21024 Public methods of transaction manager might throw
NodeStoppingException (#2938)
---
.../apache/ignite/internal/util/IgniteUtils.java | 20 ++
.../ignite/internal/tx/impl/TxManagerImpl.java | 18 -
.../tx/impl/VolatileTxStateMetaStorage.java| 43 --
3 files changed, 36 insertions(+), 45 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index edb2664650..1f9bf4e85f 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -61,6 +61,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
+import java.util.function.IntSupplier;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Stream;
@@ -829,6 +830,25 @@ public class IgniteUtils {
}
}
+/**
+ * Method that runs the provided {@code fn} in {@code busyLock}.
+ *
+ * @param busyLock Component's busy lock.
+ * @param fn Function to run.
+ * @return Result of the provided function.
+ * @throws IgniteInternalException with cause {@link
NodeStoppingException} if {@link IgniteSpinBusyLock#enterBusy()} failed.
+ */
+public static int inBusyLock(IgniteSpinBusyLock busyLock, IntSupplier fn) {
+if (!busyLock.enterBusy()) {
+throw new IgniteInternalException(NODE_STOPPING_ERR, new
NodeStoppingException());
+}
+try {
+return fn.getAsInt();
+} finally {
+busyLock.leaveBusy();
+}
+}
+
/**
* Method that runs the provided {@code fn} in {@code busyLock}.
*
diff --git
a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxManagerImpl.java
b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxManagerImpl.java
index 67ad05a466..97f0172826 100644
---
a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxManagerImpl.java
+++
b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxManagerImpl.java
@@ -29,6 +29,7 @@ import static
org.apache.ignite.internal.tx.TxState.checkTransitionCorrectness;
import static org.apache.ignite.internal.tx.TxState.isFinalState;
import static
org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
import static org.apache.ignite.internal.util.ExceptionUtils.withCause;
+import static org.apache.ignite.internal.util.IgniteUtils.inBusyLock;
import static org.apache.ignite.internal.util.IgniteUtils.inBusyLockAsync;
import static
org.apache.ignite.internal.util.IgniteUtils.shutdownAndAwaitTermination;
import static
org.apache.ignite.lang.ErrorGroups.Replicator.REPLICA_UNAVAILABLE_ERR;
@@ -276,12 +277,12 @@ public class TxManagerImpl implements TxManager,
NetworkMessageHandler {
@Override
public TxStateMeta stateMeta(UUID txId) {
-return txStateVolatileStorage.state(txId);
+return inBusyLock(busyLock, () -> txStateVolatileStorage.state(txId));
}
@Override
public TxStateMeta updateTxMeta(UUID txId, Function updater) {
-return txStateVolatileStorage.updateMeta(txId, oldMeta -> {
+return inBusyLock(busyLock, () ->
txStateVolatileStorage.updateMeta(txId, oldMeta -> {
TxStateMeta newMeta = updater.apply(oldMeta);
if (newMeta == null) {
@@ -291,7 +292,7 @@ public class TxManagerImpl implements TxManager,
NetworkMessageHandler {
TxState oldState = oldMeta == null ? null : oldMeta.txState();
return checkTransitionCorrectness(oldState, newMeta.txState()) ?
newMeta : oldMeta;
-});
+}));
}
@Override
@@ -556,16 +557,16 @@ public class TxManagerImpl implements TxManager,
NetworkMessageHandler {
@Override
public int finished() {
-return (int) txStateVolatileStorage.states().stream()
+return inBusyLock(busyLock, () -> (int)
txStateVolatileStorage.states().stream()
.filter(e -> isFinalState(e.txState()))
-.count();
+.count());
}
@Override
public int pending() {
-return (int)
(ignite) branch master updated: IGNITE-21046 Move binaryMetadataUpdateListener to basic interface (#11049)
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new e298242654e IGNITE-21046 Move binaryMetadataUpdateListener to basic
interface (#11049)
e298242654e is described below
commit e298242654e2fa56ea43d2e188f08a2b5634f2f7
Author: Maksim Timonin
AuthorDate: Fri Dec 8 15:29:19 2023 +0300
IGNITE-21046 Move binaryMetadataUpdateListener to basic interface (#11049)
---
.../processors/cache/binary/CacheObjectBinaryProcessorImpl.java | 6 ++
.../processors/cacheobject/IgniteCacheObjectProcessor.java| 8
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 7ca52e66449..4e1bc0b59c8 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -372,10 +372,8 @@ public class CacheObjectBinaryProcessorImpl extends
GridProcessorAdapter impleme
}
}
-/**
- * @param lsnr Listener.
- */
-public void addBinaryMetadataUpdateListener(BinaryMetadataUpdatedListener
lsnr) {
+/** {@inheritDoc} */
+@Override public void
addBinaryMetadataUpdateListener(BinaryMetadataUpdatedListener lsnr) {
if (transport != null)
transport.addBinaryMetadataUpdateListener(lsnr);
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
index 68953161430..1b3cd9bcc88 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
@@ -39,6 +39,7 @@ import
org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.IncompleteCacheObject;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import
org.apache.ignite.internal.processors.cache.binary.BinaryMetadataUpdatedListener;
import org.jetbrains.annotations.Nullable;
/**
@@ -377,4 +378,11 @@ public interface IgniteCacheObjectProcessor extends
GridProcessor {
* @throws org.apache.ignite.binary.BinaryObjectException In case of error.
*/
public BinaryType registerClass(Class cls) throws BinaryObjectException;
+
+/**
+ * Add binary metadata update listener.
+ *
+ * @param lsnr Listener.
+ */
+public void addBinaryMetadataUpdateListener(BinaryMetadataUpdatedListener
lsnr);
}
(ignite) branch master updated: IGNITE-21041 Make FilteredRecord local for RecordSerializer (#11082)
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new dbe7e2513d3 IGNITE-21041 Make FilteredRecord local for
RecordSerializer (#11082)
dbe7e2513d3 is described below
commit dbe7e2513d30aff3f42df618d88dba70a721e229
Author: Maksim Timonin
AuthorDate: Fri Dec 8 15:23:42 2023 +0300
IGNITE-21041 Make FilteredRecord local for RecordSerializer (#11082)
---
.../apache/ignite/internal/pagemem/wal/record/FilteredRecord.java| 3 ---
.../cache/persistence/wal/reader/StandaloneWalRecordsIterator.java | 5 -
.../cache/persistence/wal/serializer/RecordV1Serializer.java | 5 -
.../cache/persistence/wal/serializer/RecordV2Serializer.java | 5 -
4 files changed, 12 insertions(+), 6 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
index e077e5accc2..519e825953d 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
@@ -24,9 +24,6 @@ import
org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRe
* {@link AbstractWalRecordsIterator}.
*/
public class FilteredRecord extends WALRecord {
-/** Instance. */
-public static final FilteredRecord INSTANCE = new FilteredRecord();
-
/** {@inheritDoc} */
@Override public RecordType type() {
return null;
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index c7582b7cf14..26dd7c3e647 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -96,6 +96,9 @@ class StandaloneWalRecordsIterator extends
AbstractWalRecordsIterator {
/** Replay from bound include. */
private final WALPointer lowBound;
+/** Singleton instance of {@link FilteredRecord} */
+private final WALRecord filteredRecord = new FilteredRecord();
+
/**
* Creates iterator in file-by-file iteration mode. Directory
*
@@ -286,7 +289,7 @@ class StandaloneWalRecordsIterator extends
AbstractWalRecordsIterator {
return tup;
if (!checkBounds(tup.get1()))
-return new T2<>(tup.get1(), FilteredRecord.INSTANCE); //
FilteredRecord for mark as filtered.
+return new T2<>(tup.get1(), filteredRecord); // FilteredRecord for
mark as filtered.
return tup;
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
index 295e174abf8..52c360d47eb 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -98,6 +98,9 @@ public class RecordV1Serializer implements RecordSerializer {
*/
private final boolean marshalledMode;
+/** Singleton instance of {@link FilteredRecord} */
+private final FilteredRecord filteredRecord = new FilteredRecord();
+
/** Thread-local heap byte buffer. */
private final ThreadLocal heapTlb = new
ThreadLocal() {
@Override protected ByteBuffer initialValue() {
@@ -148,7 +151,7 @@ public class RecordV1Serializer implements RecordSerializer
{
if (recType.purpose() != WALRecord.RecordPurpose.INTERNAL
&& recordFilter != null && !recordFilter.apply(rec.type(),
ptr))
-return FilteredRecord.INSTANCE;
+return filteredRecord;
else if (marshalledMode) {
ByteBuffer buf = heapTlb.get();
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
index 60c68c13f7c..21aa52a91ba 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
+++
(ignite) branch master updated (dbe7e2513d3 -> ace734e6327)
This is an automated email from the ASF dual-hosted git repository. timoninmaxim pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git from dbe7e2513d3 IGNITE-21041 Make FilteredRecord local for RecordSerializer (#11082) add ace734e6327 IGNITE-17700 Refactoring WalRecordsConsumer (#11050) No new revisions were added by this update. Summary of changes: .../java/org/apache/ignite/internal/cdc/CdcMain.java | 2 +- .../ignite/internal/cdc/WalRecordsConsumer.java | 20 2 files changed, 13 insertions(+), 9 deletions(-)
(ignite) branch master updated: IGNITE-21042 Move CreateDumpFutureTask#toLong to IgniteUtils (#11048)
This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 27d9298de8f IGNITE-21042 Move CreateDumpFutureTask#toLong to
IgniteUtils (#11048)
27d9298de8f is described below
commit 27d9298de8f2e76a6fdeb7cf5b6d065b1a6c652e
Author: Maksim Timonin
AuthorDate: Fri Dec 8 15:16:57 2023 +0300
IGNITE-21042 Move CreateDumpFutureTask#toLong to IgniteUtils (#11048)
---
.../cache/persistence/snapshot/dump/CreateDumpFutureTask.java | 8 ++--
.../java/org/apache/ignite/internal/util/IgniteUtils.java | 11 +++
.../persistence/snapshot/dump/AbstractCacheDumpTest.java | 2 +-
3 files changed, 14 insertions(+), 7 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
index d0654e719b0..3f181b71a97 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
@@ -75,6 +75,7 @@ import static
org.apache.ignite.internal.processors.cache.persistence.file.FileP
import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_GRP_DIR_PREFIX;
import static
org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.DUMP_LOCK;
import static
org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.Dump.dumpPartFileName;
+import static org.apache.ignite.internal.util.IgniteUtils.toLong;
/**
* Task creates cache group dump.
@@ -111,7 +112,7 @@ public class CreateDumpFutureTask extends
AbstractCreateSnapshotFutureTask imple
* Dump contextes.
* Key is [group_id, partition_id] combined in single long value.
*
- * @see #toLong(int, int)
+ * @see IgniteUtils#toLong(int, int)
*/
private final Map dumpCtxs = new
ConcurrentHashMap<>();
@@ -631,11 +632,6 @@ public class CreateDumpFutureTask extends
AbstractCreateSnapshotFutureTask imple
}
}
-/** */
-public static long toLong(int high, int low) {
-return (((long)high) << Integer.SIZE) | (low & 0xL);
-}
-
/** */
private File groupDirectory(CacheGroupContext grpCtx) throws
IgniteCheckedException {
return new File(
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 2f736091ed1..de17a4eea7f 100755
---
a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -7704,6 +7704,17 @@ public abstract class IgniteUtils {
return ret;
}
+/**
+ * Concats two integers to long.
+ *
+ * @param high Highest bits.
+ * @param low Lowest bits.
+ * @return Long.
+ */
+public static long toLong(int high, int low) {
+return (((long)high) << Integer.SIZE) | (low & 0xL);
+}
+
/**
* Copies all elements from collection to array and asserts that
* array is big enough to hold the collection. This method should
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java
index 75d9bdda06b..bdcfab73f25 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java
@@ -79,7 +79,7 @@ import static
org.apache.ignite.internal.encryption.AbstractEncryptionTest.KEYST
import static
org.apache.ignite.internal.encryption.AbstractEncryptionTest.KEYSTORE_PATH;
import static
org.apache.ignite.internal.processors.cache.GridCacheUtils.UTILITY_CACHE_NAME;
import static
org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.SNP_RUNNING_DIR_KEY;
-import static
org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.CreateDumpFutureTask.toLong;
+import static org.apache.ignite.internal.util.IgniteUtils.toLong;
import static org.apache.ignite.platform.model.AccessLevel.SUPER;
import static org.apache.ignite.testframework.GridTestUtils.runAsync;
import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
(ignite-3) branch main updated: IGNITE-21043 Introduce a method to obtain a list from CompletableFuture#allOf (#2936)
This is an automated email from the ASF dual-hosted git repository.
apolovtsev pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 23238d8222 IGNITE-21043 Introduce a method to obtain a list from
CompletableFuture#allOf (#2936)
23238d8222 is described below
commit 23238d822205d0c8f3c44240dba78d3e5f0f4d44
Author: Alexander Polovtcev
AuthorDate: Fri Dec 8 14:16:58 2023 +0200
IGNITE-21043 Introduce a method to obtain a list from
CompletableFuture#allOf (#2936)
---
.../deployunit/DeployMessagingService.java | 44 +++
.../internal/compute/loader/JobContextManager.java | 63 ++
.../ignite/internal/util/CompletableFutures.java | 24 +
.../internal/util/CompletableFuturesTest.java | 28 ++
.../runner/app/AbstractSchemaChangeTest.java | 12 ++---
.../exec/ExecutionDependencyResolverImpl.java | 3 +-
.../internal/table/distributed/TableManager.java | 5 +-
.../replicator/PartitionReplicaListener.java | 32 ++-
8 files changed, 116 insertions(+), 95 deletions(-)
diff --git
a/modules/code-deployment/src/main/java/org/apache/ignite/internal/deployunit/DeployMessagingService.java
b/modules/code-deployment/src/main/java/org/apache/ignite/internal/deployunit/DeployMessagingService.java
index 6ef69eb630..397a8f8907 100644
---
a/modules/code-deployment/src/main/java/org/apache/ignite/internal/deployunit/DeployMessagingService.java
+++
b/modules/code-deployment/src/main/java/org/apache/ignite/internal/deployunit/DeployMessagingService.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.deployunit;
+import static java.util.concurrent.CompletableFuture.allOf;
+
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
@@ -24,13 +26,10 @@ import org.apache.ignite.compute.version.Version;
import
org.apache.ignite.internal.cluster.management.ClusterManagementGroupManager;
import
org.apache.ignite.internal.deployunit.exception.DeploymentUnitReadException;
import org.apache.ignite.internal.deployunit.message.DeployUnitMessageTypes;
+import org.apache.ignite.internal.deployunit.message.DeploymentUnitFactory;
import org.apache.ignite.internal.deployunit.message.DownloadUnitRequest;
-import org.apache.ignite.internal.deployunit.message.DownloadUnitRequestImpl;
import org.apache.ignite.internal.deployunit.message.DownloadUnitResponse;
-import org.apache.ignite.internal.deployunit.message.DownloadUnitResponseImpl;
import org.apache.ignite.internal.deployunit.message.StopDeployRequest;
-import org.apache.ignite.internal.deployunit.message.StopDeployRequestImpl;
-import org.apache.ignite.internal.deployunit.message.StopDeployResponseImpl;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
import org.apache.ignite.network.ChannelType;
@@ -65,6 +64,8 @@ public class DeployMessagingService {
*/
private final DownloadTracker tracker;
+private final DeploymentUnitFactory messageFactory = new
DeploymentUnitFactory();
+
/**
* Constructor.
*
@@ -108,7 +109,7 @@ public class DeployMessagingService {
* @return Downloaded deployment unit content.
*/
CompletableFuture downloadUnitContent(String id, Version
version, List nodes) {
-DownloadUnitRequest request = DownloadUnitRequestImpl.builder()
+DownloadUnitRequest request = messageFactory.downloadUnitRequest()
.id(id)
.version(version.render())
.build();
@@ -137,34 +138,35 @@ public class DeployMessagingService {
* @param version Deployment unit version.
* @return Future with stop result.
*/
-public CompletableFuture stopInProgressDeploy(String id, Version
version) {
+CompletableFuture stopInProgressDeploy(String id, Version version) {
LOG.info("Stop in progress deploy for " + id + ":" + version);
-return CompletableFuture.allOf(cmgManager.logicalTopology()
-.thenApply(topology -> topology.nodes().stream().map(node ->
-clusterService.messagingService()
-.invoke(node,
-DEPLOYMENT_CHANNEL,
-StopDeployRequestImpl
-.builder()
-.id(id)
-
.version(version.render())
-.build(),
-Long.MAX_VALUE)
-).toArray(CompletableFuture[]::new)));
+
+return cmgManager.logicalTopology()
+.thenCompose(topology -> {
+
(ignite) 01/03: IGNITE-20614 Add rate limiter for dumps (#11076)
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch ignite-2.16
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 0547e7fd688c49547e907224c2a9bfa2c1f0bda5
Author: Maksim Timonin
AuthorDate: Wed Dec 6 10:34:42 2023 +0300
IGNITE-20614 Add rate limiter for dumps (#11076)
Co-authored-by: nizhi...@apache.org
(cherry picked from commit 65aa55af1707e22db8fb702204016c1c08ae0bfa)
---
.../snapshot/IgniteSnapshotManager.java| 3 +-
.../snapshot/dump/CreateDumpFutureTask.java| 19
.../snapshot/dump/IgniteCacheDumpSelf2Test.java| 50 ++
3 files changed, 71 insertions(+), 1 deletion(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
index 47e4e1c7f44..4890ab85b57 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
@@ -2754,6 +2754,7 @@ public class IgniteSnapshotManager extends
GridCacheSharedManagerAdapter
snpName,
snapshotLocalDir(snpName, snpPath),
ioFactory,
+transferRateLimiter,
snpSndr,
parts,
compress
@@ -2816,7 +2817,7 @@ public class IgniteSnapshotManager extends
GridCacheSharedManagerAdapter
}
/** @return Current snapshot task. */
-> T currentSnapshotTask(Class
snpTaskCls) {
+public > T
currentSnapshotTask(Class snpTaskCls) {
SnapshotOperationRequest req = clusterSnpReq;
if (req == null)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
index 7e80a8dc64d..d57f755a8d4 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java
@@ -61,6 +61,7 @@ import
org.apache.ignite.internal.processors.cache.persistence.snapshot.Snapshot
import
org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotSender;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import
org.apache.ignite.internal.processors.cache.version.GridCacheVersionManager;
+import org.apache.ignite.internal.util.BasicRateLimiter;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.lang.GridCloseableIterator;
@@ -96,6 +97,12 @@ public class CreateDumpFutureTask extends
AbstractCreateSnapshotFutureTask imple
/** If {@code true} then compress partition files. */
private final boolean compress;
+/** Dump transfer rate limiter. */
+private final BasicRateLimiter rateLimiter;
+
+/** Processed dump size in bytes. */
+private final AtomicLong processedSize = new AtomicLong();
+
/**
* Dump contextes.
* Key is [group_id, partition_id] combined in single long value.
@@ -122,6 +129,7 @@ public class CreateDumpFutureTask extends
AbstractCreateSnapshotFutureTask imple
* @param dumpName Dump name.
* @param ioFactory IO factory.
* @param snpSndr Snapshot sender.
+ * @param rateLimiter Dump transfer rate limiter.
* @param parts Parts to dump.
* @param compress If {@code true} then compress partition files.
*/
@@ -132,6 +140,7 @@ public class CreateDumpFutureTask extends
AbstractCreateSnapshotFutureTask imple
String dumpName,
File dumpDir,
FileIOFactory ioFactory,
+BasicRateLimiter rateLimiter,
SnapshotSender snpSndr,
Map> parts,
boolean compress
@@ -148,6 +157,7 @@ public class CreateDumpFutureTask extends
AbstractCreateSnapshotFutureTask imple
this.dumpDir = dumpDir;
this.ioFactory = compress ? new WriteOnlyZipFileIOFactory() :
ioFactory;
this.compress = compress;
+this.rateLimiter = rateLimiter;
}
/** {@inheritDoc} */
@@ -343,6 +353,11 @@ public class CreateDumpFutureTask extends
AbstractCreateSnapshotFutureTask imple
return closeFut;
}
+/** @return Processed dump size in bytes. */
+public long processedSize() {
+return processedSize.get();
+}
+
/** */
private void clearDumpListener(CacheGroupContext gctx) {
for (GridCacheContext cctx :
(ignite) 03/03: [MINOR] Fix error message in test
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch ignite-2.16
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit d18d899ba2e4bf16ceebb9e2da3a50a19624718e
Author: Nikolay
AuthorDate: Fri Dec 8 14:28:45 2023 +0300
[MINOR] Fix error message in test
(cherry picked from commit 233c8bd2fb7176309f88ef2cdab658d6710c787e)
---
.../cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java
index 44b7927d73a..5ccaea42e3f 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java
@@ -714,7 +714,7 @@ public class IgniteCacheDumpSelf2Test extends
GridCommonAbstractTest {
).run();
},
IgniteException.class,
-"Dump '" + DMP_NAME + "' has different master key digest. To
restore this snapshot, provide the same master key"
+"Dump '" + DMP_NAME + "' has different master key digest"
);
Map dumpEntries = new HashMap<>();
(ignite) branch ignite-2.16 updated (1fd546f754e -> d18d899ba2e)
This is an automated email from the ASF dual-hosted git repository. nizhikov pushed a change to branch ignite-2.16 in repository https://gitbox.apache.org/repos/asf/ignite.git from 1fd546f754e IGNITE-19436 Fixed jvmdefaults.bat on JDK 17 (#10707) new 0547e7fd688 IGNITE-20614 Add rate limiter for dumps (#11076) new eae3bc33b9e IGNITE-21013 Support encryption of cache dumps (#11078) new d18d899ba2e [MINOR] Fix error message in test The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/ignite/dump/DumpReader.java| 19 +- .../ignite/dump/DumpReaderConfiguration.java | 19 +- .../snapshot/IgniteSnapshotManager.java| 53 +++-- .../persistence/snapshot/SnapshotMetadata.java | 14 +- .../snapshot/SnapshotOperationRequest.java | 13 +- .../snapshot/SnapshotPartitionsVerifyHandler.java | 5 +- .../snapshot/SnapshotRestoreProcess.java | 1 + .../snapshot/dump/CreateDumpFutureTask.java| 54 - .../cache/persistence/snapshot/dump/Dump.java | 43 +++- .../snapshot/dump/DumpEntrySerializer.java | 220 - .../snapshot/AbstractSnapshotSelfTest.java | 1 + .../snapshot/EncryptedSnapshotTest.java| 1 + .../snapshot/IgniteSnapshotManagerSelfTest.java| 1 + .../snapshot/dump/AbstractCacheDumpTest.java | 37 +++- .../snapshot/dump/IgniteCacheDumpSelf2Test.java| 187 +- .../dump/IgniteConcurrentCacheDumpTest.java| 13 +- 16 files changed, 587 insertions(+), 94 deletions(-)
(ignite) 02/03: IGNITE-21013 Support encryption of cache dumps (#11078)
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch ignite-2.16
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit eae3bc33b9e0a4c7d90b9df7d3bfd631b520132e
Author: Nikolay
AuthorDate: Fri Dec 8 14:28:45 2023 +0300
IGNITE-21013 Support encryption of cache dumps (#11078)
(cherry picked from commit 668939affaeced34b59f0aa95bedd53f3a5bfe6a)
---
.../java/org/apache/ignite/dump/DumpReader.java| 19 +-
.../ignite/dump/DumpReaderConfiguration.java | 19 +-
.../snapshot/IgniteSnapshotManager.java| 50 -
.../persistence/snapshot/SnapshotMetadata.java | 14 +-
.../snapshot/SnapshotOperationRequest.java | 13 +-
.../snapshot/SnapshotPartitionsVerifyHandler.java | 5 +-
.../snapshot/SnapshotRestoreProcess.java | 1 +
.../snapshot/dump/CreateDumpFutureTask.java| 35 +++-
.../cache/persistence/snapshot/dump/Dump.java | 43 +++-
.../snapshot/dump/DumpEntrySerializer.java | 220 -
.../snapshot/AbstractSnapshotSelfTest.java | 1 +
.../snapshot/EncryptedSnapshotTest.java| 1 +
.../snapshot/IgniteSnapshotManagerSelfTest.java| 1 +
.../snapshot/dump/AbstractCacheDumpTest.java | 37 +++-
.../snapshot/dump/IgniteCacheDumpSelf2Test.java| 137 -
.../dump/IgniteConcurrentCacheDumpTest.java| 13 +-
16 files changed, 516 insertions(+), 93 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java
b/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java
index d2ed811ac3f..bb6f8cfb405 100644
--- a/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java
@@ -41,6 +41,8 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteExperimental;
+import org.apache.ignite.spi.IgniteSpiAdapter;
+import org.apache.ignite.spi.encryption.EncryptionSpi;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static
org.apache.ignite.configuration.DataStorageConfiguration.DFLT_MARSHALLER_PATH;
@@ -75,7 +77,7 @@ public class DumpReader implements Runnable {
@Override public void run() {
ackAsciiLogo();
-try (Dump dump = new Dump(cfg.dumpRoot(), cfg.keepBinary(), false,
log)) {
+try (Dump dump = new Dump(cfg.dumpRoot(), null, cfg.keepBinary(),
false, encryptionSpi(), log)) {
DumpConsumer cnsmr = cfg.consumer();
cnsmr.start();
@@ -231,4 +233,19 @@ public class DumpReader implements Runnable {
"");
}
}
+
+/** */
+private EncryptionSpi encryptionSpi() {
+EncryptionSpi encSpi = cfg.encryptionSpi();
+
+if (encSpi == null)
+return null;
+
+if (encSpi instanceof IgniteSpiAdapter)
+((IgniteSpiAdapter)encSpi).onBeforeStart();
+
+encSpi.spiStart("dump-reader");
+
+return encSpi;
+}
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java
b/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java
index a7a80b8227f..619c532b1ff 100644
---
a/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java
+++
b/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java
@@ -22,6 +22,7 @@ import java.time.Duration;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.lang.IgniteExperimental;
+import org.apache.ignite.spi.encryption.EncryptionSpi;
/**
* Configuration class of {@link DumpReader}.
@@ -61,12 +62,15 @@ public class DumpReaderConfiguration {
/** Skip copies. */
private final boolean skipCopies;
+/** Encryption SPI. */
+private final EncryptionSpi encSpi;
+
/**
* @param dir Root dump directory.
* @param cnsmr Dump consumer.
*/
public DumpReaderConfiguration(File dir, DumpConsumer cnsmr) {
-this(dir, cnsmr, DFLT_THREAD_CNT, DFLT_TIMEOUT, true, true, null,
false);
+this(dir, cnsmr, DFLT_THREAD_CNT, DFLT_TIMEOUT, true, true, null,
false, null);
}
/**
@@ -78,15 +82,18 @@ public class DumpReaderConfiguration {
* @param keepBinary If {@code true} then don't deserialize {@link
KeyCacheObject} and {@link CacheObject}.
* @param cacheGroupNames Cache group names.
* @param skipCopies Skip copies.
+ * @param encSpi Encryption SPI.
*/
-public DumpReaderConfiguration(File dir,
+public DumpReaderConfiguration(
+File dir,
DumpConsumer cnsmr,
int thCnt,
Duration timeout,
boolean failFast,
boolean
(ignite) branch master updated: [MINOR] Fix error message in test
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 233c8bd2fb7 [MINOR] Fix error message in test
233c8bd2fb7 is described below
commit 233c8bd2fb7176309f88ef2cdab658d6710c787e
Author: Nikolay
AuthorDate: Fri Dec 8 14:28:45 2023 +0300
[MINOR] Fix error message in test
---
.../cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java
index 44b7927d73a..5ccaea42e3f 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java
@@ -714,7 +714,7 @@ public class IgniteCacheDumpSelf2Test extends
GridCommonAbstractTest {
).run();
},
IgniteException.class,
-"Dump '" + DMP_NAME + "' has different master key digest. To
restore this snapshot, provide the same master key"
+"Dump '" + DMP_NAME + "' has different master key digest"
);
Map dumpEntries = new HashMap<>();
(ignite) branch master updated: IGNITE-21013 Support encryption of cache dumps (#11078)
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 668939affae IGNITE-21013 Support encryption of cache dumps (#11078)
668939affae is described below
commit 668939affaeced34b59f0aa95bedd53f3a5bfe6a
Author: Nikolay
AuthorDate: Fri Dec 8 14:28:45 2023 +0300
IGNITE-21013 Support encryption of cache dumps (#11078)
---
.../java/org/apache/ignite/dump/DumpReader.java| 19 +-
.../ignite/dump/DumpReaderConfiguration.java | 19 +-
.../snapshot/IgniteSnapshotManager.java| 50 -
.../persistence/snapshot/SnapshotMetadata.java | 14 +-
.../snapshot/SnapshotOperationRequest.java | 13 +-
.../snapshot/SnapshotPartitionsVerifyHandler.java | 5 +-
.../snapshot/SnapshotRestoreProcess.java | 1 +
.../snapshot/dump/CreateDumpFutureTask.java| 35 +++-
.../cache/persistence/snapshot/dump/Dump.java | 43 +++-
.../snapshot/dump/DumpEntrySerializer.java | 220 -
.../snapshot/AbstractSnapshotSelfTest.java | 1 +
.../snapshot/EncryptedSnapshotTest.java| 1 +
.../snapshot/IgniteSnapshotManagerSelfTest.java| 1 +
.../snapshot/dump/AbstractCacheDumpTest.java | 37 +++-
.../snapshot/dump/IgniteCacheDumpSelf2Test.java| 137 -
.../dump/IgniteConcurrentCacheDumpTest.java| 13 +-
16 files changed, 516 insertions(+), 93 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java
b/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java
index d2ed811ac3f..bb6f8cfb405 100644
--- a/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java
@@ -41,6 +41,8 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteExperimental;
+import org.apache.ignite.spi.IgniteSpiAdapter;
+import org.apache.ignite.spi.encryption.EncryptionSpi;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static
org.apache.ignite.configuration.DataStorageConfiguration.DFLT_MARSHALLER_PATH;
@@ -75,7 +77,7 @@ public class DumpReader implements Runnable {
@Override public void run() {
ackAsciiLogo();
-try (Dump dump = new Dump(cfg.dumpRoot(), cfg.keepBinary(), false,
log)) {
+try (Dump dump = new Dump(cfg.dumpRoot(), null, cfg.keepBinary(),
false, encryptionSpi(), log)) {
DumpConsumer cnsmr = cfg.consumer();
cnsmr.start();
@@ -231,4 +233,19 @@ public class DumpReader implements Runnable {
"");
}
}
+
+/** */
+private EncryptionSpi encryptionSpi() {
+EncryptionSpi encSpi = cfg.encryptionSpi();
+
+if (encSpi == null)
+return null;
+
+if (encSpi instanceof IgniteSpiAdapter)
+((IgniteSpiAdapter)encSpi).onBeforeStart();
+
+encSpi.spiStart("dump-reader");
+
+return encSpi;
+}
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java
b/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java
index a7a80b8227f..619c532b1ff 100644
---
a/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java
+++
b/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java
@@ -22,6 +22,7 @@ import java.time.Duration;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.lang.IgniteExperimental;
+import org.apache.ignite.spi.encryption.EncryptionSpi;
/**
* Configuration class of {@link DumpReader}.
@@ -61,12 +62,15 @@ public class DumpReaderConfiguration {
/** Skip copies. */
private final boolean skipCopies;
+/** Encryption SPI. */
+private final EncryptionSpi encSpi;
+
/**
* @param dir Root dump directory.
* @param cnsmr Dump consumer.
*/
public DumpReaderConfiguration(File dir, DumpConsumer cnsmr) {
-this(dir, cnsmr, DFLT_THREAD_CNT, DFLT_TIMEOUT, true, true, null,
false);
+this(dir, cnsmr, DFLT_THREAD_CNT, DFLT_TIMEOUT, true, true, null,
false, null);
}
/**
@@ -78,15 +82,18 @@ public class DumpReaderConfiguration {
* @param keepBinary If {@code true} then don't deserialize {@link
KeyCacheObject} and {@link CacheObject}.
* @param cacheGroupNames Cache group names.
* @param skipCopies Skip copies.
+ * @param encSpi Encryption SPI.
*/
-public DumpReaderConfiguration(File dir,
+public DumpReaderConfiguration(
+File dir,
DumpConsumer cnsmr,
(ignite-3) branch ignite-15571 created (now d45894db62)
This is an automated email from the ASF dual-hosted git repository. zstan pushed a change to branch ignite-15571 in repository https://gitbox.apache.org/repos/asf/ignite-3.git at d45894db62 IGNITE-15571 Sql. Remove mention of already fixed issue This branch includes the following new commits: new d45894db62 IGNITE-15571 Sql. Remove mention of already fixed issue The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(ignite-3) 01/01: IGNITE-15571 Sql. Remove mention of already fixed issue
This is an automated email from the ASF dual-hosted git repository. zstan pushed a commit to branch ignite-15571 in repository https://gitbox.apache.org/repos/asf/ignite-3.git commit d45894db62ad489236b36e2cf6de26784058eed0 Author: zstan AuthorDate: Fri Dec 8 11:05:36 2023 +0300 IGNITE-15571 Sql. Remove mention of already fixed issue --- .../src/integrationTest/sql/join/inner/test_using_join.test_ignore | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/sql-engine/src/integrationTest/sql/join/inner/test_using_join.test_ignore b/modules/sql-engine/src/integrationTest/sql/join/inner/test_using_join.test_ignore index 004f5ee964..3b20ea8729 100644 --- a/modules/sql-engine/src/integrationTest/sql/join/inner/test_using_join.test_ignore +++ b/modules/sql-engine/src/integrationTest/sql/join/inner/test_using_join.test_ignore @@ -2,7 +2,6 @@ # description: Test USING joins # group: [inner] # Ignore https://issues.apache.org/jira/browse/IGNITE-19125 -# Ignore https://issues.apache.org/jira/browse/IGNITE-15571 # Ignore https://issues.apache.org/jira/browse/IGNITE-15570 statement ok
