Merge.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5b2400ab Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5b2400ab Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5b2400ab Branch: refs/heads/ignite-2.1.2-exchange Commit: 5b2400ab4a0c43a1b8597fd674d5c6a373bd3c43 Parents: 377cc9d aa21a9b Author: sboikov <[email protected]> Authored: Wed Jul 5 14:22:31 2017 +0300 Committer: sboikov <[email protected]> Committed: Wed Jul 5 14:22:31 2017 +0300 ---------------------------------------------------------------------- .../jdbc2/JdbcPreparedStatementSelfTest.java | 35 + .../jdbc/JdbcPreparedStatementSelfTest.java | 35 + .../thin/JdbcThinPreparedStatementSelfTest.java | 35 + .../PersistentStoreConfiguration.java | 39 +- .../org/apache/ignite/events/EventType.java | 12 + .../ignite/events/WalSegmentArchivedEvent.java | 62 + .../apache/ignite/internal/GridComponent.java | 4 +- .../ignite/internal/GridPluginComponent.java | 2 +- .../apache/ignite/internal/IgniteKernal.java | 33 +- .../internal/jdbc/JdbcPreparedStatement.java | 6 +- .../internal/jdbc/thin/JdbcThinConnection.java | 7 +- .../jdbc/thin/JdbcThinPreparedStatement.java | 2 - .../internal/jdbc2/JdbcPreparedStatement.java | 12 +- .../internal/managers/GridManagerAdapter.java | 2 +- .../checkpoint/GridCheckpointManager.java | 2 +- .../managers/communication/GridIoManager.java | 10 +- .../communication/GridMessageListener.java | 3 +- .../deployment/GridDeploymentCommunication.java | 4 +- .../internal/managers/discovery/DiscoCache.java | 17 +- .../discovery/DiscoveryLocalJoinData.java | 104 ++ .../discovery/GridDiscoveryManager.java | 128 +- .../eventstorage/GridEventStorageManager.java | 4 +- .../pagemem/store/IgnitePageStoreManager.java | 3 +- .../internal/pagemem/wal/record/WALRecord.java | 11 +- .../processors/GridProcessorAdapter.java | 2 +- .../cache/CacheAffinitySharedManager.java | 67 +- .../processors/cache/CacheGroupContext.java | 4 +- .../processors/cache/CacheGroupData.java | 4 +- .../cache/ChangeGlobalStateMessage.java | 120 -- .../processors/cache/ClusterCachesInfo.java | 493 +++++-- .../internal/processors/cache/ClusterState.java | 38 - .../cache/DynamicCacheChangeRequest.java | 52 +- .../processors/cache/ExchangeActions.java | 37 +- .../processors/cache/GridCacheEventManager.java | 2 - .../cache/GridCacheEvictionManager.java | 1 - .../processors/cache/GridCacheIoManager.java | 98 +- .../processors/cache/GridCacheMvccManager.java | 9 +- .../GridCachePartitionExchangeManager.java | 439 +++--- .../processors/cache/GridCacheProcessor.java | 193 +-- .../cache/GridCacheSharedContext.java | 60 +- .../cache/GridCacheSharedManager.java | 6 - .../cache/GridCacheSharedManagerAdapter.java | 16 - .../cache/GridDeferredAckMessageSender.java | 3 +- .../processors/cache/PendingDiscoveryEvent.java | 61 + .../processors/cache/StateChangeRequest.java | 77 ++ .../cache/binary/BinaryMetadataTransport.java | 4 +- .../binary/CacheObjectBinaryProcessorImpl.java | 4 +- .../distributed/GridCacheTxRecoveryFuture.java | 1 - .../distributed/dht/GridDhtCacheAdapter.java | 1 - .../cache/distributed/dht/GridDhtGetFuture.java | 1 - .../distributed/dht/GridDhtGetSingleFuture.java | 2 - .../dht/GridDhtPartitionTopologyImpl.java | 13 +- .../dht/GridDhtTopologyFutureAdapter.java | 2 +- .../dht/GridPartitionedSingleGetFuture.java | 3 - .../GridNearAtomicAbstractUpdateFuture.java | 1 - .../dht/preloader/GridDhtForceKeysFuture.java | 1 - .../dht/preloader/GridDhtPartitionDemander.java | 2 + .../GridDhtPartitionsExchangeFuture.java | 240 +++- .../preloader/GridDhtPartitionsFullMessage.java | 44 +- .../GridDhtPartitionsSingleMessage.java | 38 +- .../dht/preloader/GridDhtPreloader.java | 2 +- .../distributed/near/GridNearGetFuture.java | 2 - .../near/GridNearTxPrepareRequest.java | 1 - .../GridCacheDatabaseSharedManager.java | 111 +- .../persistence/GridCacheOffheapManager.java | 5 +- .../IgniteCacheDatabaseSharedManager.java | 74 +- .../persistence/IgniteCacheSnapshotManager.java | 20 +- .../persistence/file/FilePageStoreManager.java | 14 +- .../persistence/tree/io/TrackingPageIO.java | 12 +- .../wal/AbstractWalRecordsIterator.java | 289 ++++ .../cache/persistence/wal/FileInput.java | 16 +- .../cache/persistence/wal/FileWALPointer.java | 4 +- .../wal/FileWriteAheadLogManager.java | 594 ++++---- .../cache/persistence/wal/RecordSerializer.java | 5 + .../persistence/wal/SegmentArchiveResult.java | 61 + .../persistence/wal/SegmentEofException.java | 3 +- .../wal/reader/IgniteWalIteratorFactory.java | 102 ++ .../wal/reader/StandaloneGridKernalContext.java | 499 +++++++ ...ndaloneIgniteCacheDatabaseSharedManager.java | 30 + .../reader/StandaloneWalRecordsIterator.java | 258 ++++ .../wal/serializer/RecordV1Serializer.java | 45 +- .../query/GridCacheDistributedQueryManager.java | 4 +- .../store/GridCacheStoreManagerAdapter.java | 1 - .../cache/transactions/IgniteTxManager.java | 2 +- .../cache/version/GridCacheVersionManager.java | 6 - .../cacheobject/IgniteCacheObjectProcessor.java | 5 - .../IgniteCacheObjectProcessorImpl.java | 5 - .../cluster/ChangeGlobalStateFinishMessage.java | 86 ++ .../cluster/ChangeGlobalStateMessage.java | 140 ++ .../processors/cluster/ClusterProcessor.java | 5 +- .../cluster/DiscoveryDataClusterState.java | 157 +++ .../cluster/GridClusterStateProcessor.java | 1129 ++++++--------- .../cluster/IgniteChangeGlobalStateSupport.java | 3 +- .../continuous/GridContinuousProcessor.java | 4 +- .../datastreamer/DataStreamProcessor.java | 2 +- .../datastreamer/DataStreamerImpl.java | 2 +- .../datastructures/DataStructuresProcessor.java | 6 +- .../datastructures/GridCacheAtomicLongImpl.java | 2 +- .../GridCacheAtomicReferenceImpl.java | 2 +- .../GridCacheAtomicSequenceImpl.java | 2 +- .../GridCacheAtomicStampedImpl.java | 2 +- .../GridCacheCountDownLatchImpl.java | 2 +- .../datastructures/GridCacheLockImpl.java | 4 +- .../datastructures/GridCacheQueueAdapter.java | 1 - .../datastructures/GridCacheSemaphoreImpl.java | 2 +- .../datastructures/GridCacheSetImpl.java | 1 - .../processors/igfs/IgfsDataManager.java | 2 +- .../igfs/IgfsFragmentizerManager.java | 4 +- .../internal/processors/igfs/IgfsImpl.java | 2 - .../internal/processors/igfs/IgfsProcessor.java | 2 +- .../processors/job/GridJobProcessor.java | 8 +- .../GridMarshallerMappingProcessor.java | 4 +- .../utils/PlatformConfigurationUtils.java | 8 +- .../processors/query/GridQueryProcessor.java | 6 +- .../processors/rest/GridRestProcessor.java | 2 +- .../cluster/GridChangeStateCommandHandler.java | 2 +- .../handlers/task/GridTaskCommandHandler.java | 4 +- .../service/GridServiceProcessor.java | 6 +- .../processors/task/GridTaskProcessor.java | 8 +- .../visor/cache/VisorCacheConfiguration.java | 11 +- .../visor/node/VisorBasicConfiguration.java | 2 +- .../node/VisorMemoryPolicyConfiguration.java | 2 +- .../visor/node/VisorNodeDataCollectorJob.java | 32 +- .../node/VisorNodeDataCollectorJobResult.java | 73 +- .../visor/node/VisorNodeDataCollectorTask.java | 14 +- .../node/VisorNodeDataCollectorTaskResult.java | 28 +- .../visor/node/VisorPersistenceMetrics.java | 214 +++ .../visor/service/VisorServiceTask.java | 10 +- .../jobstealing/JobStealingCollisionSpi.java | 2 +- .../ignite/spi/discovery/tcp/ClientImpl.java | 12 +- .../ignite/spi/discovery/tcp/ServerImpl.java | 10 +- .../internal/TestRecordingCommunicationSpi.java | 10 + ...GridManagerLocalMessageListenerSelfTest.java | 4 +- ...idCommunicationManagerListenersSelfTest.java | 2 +- .../GridCommunicationSendMessageSelfTest.java | 2 +- ...unicationBalanceMultipleConnectionsTest.java | 5 + .../cache/GridCacheAbstractMetricsSelfTest.java | 24 - .../cache/GridCachePartitionedGetSelfTest.java | 2 +- .../cache/IgniteActiveClusterTest.java | 182 --- .../IgniteClusterActivateDeactivateTest.java | 1284 ++++++++++++++++++ ...erActivateDeactivateTestWithPersistence.java | 197 +++ .../IgniteDaemonNodeMarshallerCacheTest.java | 10 - ...lerCacheClientRequestsMappingOnMissTest.java | 6 +- ...naryObjectMetadataExchangeMultinodeTest.java | 6 +- .../IgniteSemaphoreAbstractSelfTest.java | 17 +- ...IgnitePersistentStoreDataStructuresTest.java | 2 + .../wal/IgniteWalHistoryReservationsTest.java | 2 +- .../db/wal/reader/IgniteWalReaderTest.java | 385 ++++++ .../db/wal/reader/MockWalIteratorFactory.java | 114 ++ .../pagemem/NoOpPageStoreManager.java | 12 +- .../persistence/pagemem/NoOpWALManager.java | 23 +- .../AbstractNodeJoinTemplate.java | 149 +- .../IgniteChangeGlobalStateAbstractTest.java | 65 +- .../IgniteChangeGlobalStateCacheTest.java | 2 +- ...IgniteChangeGlobalStateDataStreamerTest.java | 5 +- ...gniteChangeGlobalStateDataStructureTest.java | 6 +- .../IgniteChangeGlobalStateFailOverTest.java | 26 +- .../IgniteChangeGlobalStateTest.java | 158 +-- .../IgniteStandByClusterTest.java | 17 +- .../join/JoinActiveNodeToActiveCluster.java | 62 +- ...ctiveNodeToActiveClusterWithPersistence.java | 17 + .../IgniteStandByClientReconnectTest.java | 13 +- ...eStandByClientReconnectToNewClusterTest.java | 13 +- ...DeadlockDetectionMessageMarshallingTest.java | 2 +- .../communication/GridIoManagerBenchmark.java | 4 +- .../communication/GridIoManagerBenchmark0.java | 12 +- .../communication/GridCacheMessageSelfTest.java | 2 +- ...cpCommunicationSpiMultithreadedSelfTest.java | 2 +- .../testframework/GridSpiTestContext.java | 5 +- .../testframework/junits/GridAbstractTest.java | 4 +- .../junits/common/GridCommonAbstractTest.java | 3 + .../ignite/testsuites/IgnitePdsTestSuite2.java | 9 +- .../testsuites/IgniteStandByClusterSuite.java | 5 +- .../processors/hadoop/HadoopProcessor.java | 4 +- .../hadoop/shuffle/HadoopShuffle.java | 2 +- ...ileSystemShmemExternalDualAsyncSelfTest.java | 5 + .../query/h2/opt/GridH2IndexBase.java | 2 +- .../query/h2/twostep/GridMapQueryExecutor.java | 2 +- .../h2/twostep/GridReduceQueryExecutor.java | 2 +- .../cache/IgniteCacheAbstractQuerySelfTest.java | 3 +- .../Cache/CacheConfigurationTest.cs | 21 + .../Apache.Ignite.Core.Tests/EventsTest.cs | 19 +- .../Cache/Configuration/CacheConfiguration.cs | 75 +- .../Apache.Ignite.Core/IgniteConfiguration.cs | 7 +- .../Impl/Events/RemoteListenEventFilter.cs | 3 + .../Apache.Ignite.Core/Impl/IgniteUtils.cs | 12 +- .../spark/JavaEmbeddedIgniteRDDSelfTest.java | 5 + .../frontend/app/filters/duration.filter.js | 2 +- .../generator/ConfigurationGenerator.js | 9 +- .../states/configuration/caches/affinity.pug | 18 +- .../states/configuration/caches/general.pug | 50 +- 191 files changed, 7182 insertions(+), 2830 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/5b2400ab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java ---------------------------------------------------------------------- diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 6301497,8d08c3f..a8af753 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@@ -53,7 -53,8 +54,9 @@@ import org.apache.ignite.internal.proce import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; +import org.apache.ignite.internal.util.GridAtomicLong; + import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; + import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; import org.apache.ignite.internal.util.future.GridCompoundFuture; import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; http://git-wip-us.apache.org/repos/asf/ignite/blob/5b2400ab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/5b2400ab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java ---------------------------------------------------------------------- diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index e238b11,624dec0..dab678a --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@@ -2170,9 -2148,10 +2150,12 @@@ public class GridCacheProcessor extend if (!sharedCtx.kernalContext().clientNode()) sharedCtx.database().onCacheGroupsStopped(stoppedGroups); + + if (exchActions.deactivate()) + sharedCtx.deactivate(); } + + log.info("processor onExchangeDone [topVer=" + topVer + ", time=" + (U.currentTimeMillis() - start) + ']'); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/5b2400ab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java ---------------------------------------------------------------------- diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 4e46daa,cea758a..3479b4e --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@@ -77,9 -73,9 +77,10 @@@ import org.apache.ignite.internal.proce import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; - import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor; + import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; + import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage; import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter; +import org.apache.ignite.internal.util.GridAtomicLong; import org.apache.ignite.internal.util.future.GridCompoundFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@@ -578,9 -571,12 +592,13 @@@ public class GridDhtPartitionsExchangeF if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) { DiscoveryCustomMessage msg = ((DiscoveryCustomEvent)discoEvt).customMessage(); - if (msg instanceof DynamicCacheChangeBatch) { - long start = U.currentTimeMillis(); + if (msg instanceof ChangeGlobalStateMessage) { + assert exchActions != null && !exchActions.empty(); + exchange = onClusterStateChangeRequest(crdNode); + } + else if (msg instanceof DynamicCacheChangeBatch) { ++ long start = U.currentTimeMillis(); assert exchActions != null && !exchActions.empty(); exchange = onCacheChangeRequest(crdNode); @@@ -609,12 -601,28 +627,30 @@@ cctx.affinity().initStartedCaches(crdNode, this, receivedCaches); } - else - cctx.cache().startCachesOnLocalJoin(topVer); + else { + cctx.activate(); + + List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches = + cctx.cache().cachesToStartOnLocalJoin(); + + if (cctx.database().persistenceEnabled() && + !cctx.kernalContext().clientNode()) { + List<DynamicCacheDescriptor> startDescs = new ArrayList<>(); + + if (caches != null) { + for (T2<DynamicCacheDescriptor, NearCacheConfiguration> c : caches) + startDescs.add(c.get1()); + } + + cctx.database().readCheckpointAndRestoreMemory(startDescs); + } + + cctx.cache().startCachesOnLocalJoin(caches, topVer); + } } + exchLog.info("Caches start time [topVer=" + topologyVersion() + ", time=" + (U.currentTimeMillis() - start) + ']'); + exchange = CU.clientNode(discoEvt.eventNode()) ? onClientNodeEvent(crdNode) : onServerNodeEvent(crdNode); @@@ -1227,11 -1285,10 +1333,12 @@@ assert !nodes.contains(cctx.localNode()); + sndTs = U.currentTimeMillis(); + - if (log.isDebugEnabled()) + if (log.isDebugEnabled()) { log.debug("Sending full partition map [nodeIds=" + F.viewReadOnly(nodes, F.node2id()) + ", exchId=" + exchId + ", msg=" + m + ']'); + } for (ClusterNode node : nodes) { try { @@@ -1586,18 -1568,9 +1694,18 @@@ if (updateSingleMap) { try { - // Do not update partition map, in case cluster transitioning to inactive state. + cnt.incrementAndGet(); + + long start = U.currentTimeMillis(); + - if (!exchangeOnChangeGlobalState || exchActions.newClusterState() != ClusterState.INACTIVE) + if (!deactivateCluster()) updatePartitionSingleMap(node, msg); + + long time = U.currentTimeMillis() - start; + + minTime.setIfLess(time); + maxTime.setIfGreater(time); + totTime.addAndGet(time); } finally { synchronized (this) { @@@ -2068,14 -2048,17 +2195,19 @@@ } } + rcvTs = U.currentTimeMillis(); + updatePartitionFullMap(msg); - if (exchangeOnChangeGlobalState && !F.isEmpty(msg.getExceptionsMap())) - cctx.kernalContext().state().onFullResponseMessage(msg.getExceptionsMap()); + IgniteCheckedException err = null; - onDone(exchId.topologyVersion()); + if (stateChangeExchange() && !F.isEmpty(msg.getErrorsMap())) { + err = new IgniteCheckedException("Cluster state change failed"); + + cctx.kernalContext().state().onStateChangeError(msg.getErrorsMap(), exchActions.stateChangeRequest()); + } + + onDone(exchId.topologyVersion(), err); } /**
