Merge remote-tracking branch 'remotes/origin/master' into ignite-5075-cacheStart
# Conflicts: # modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java # modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/aa0142ae Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/aa0142ae Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/aa0142ae Branch: refs/heads/ignite-5075 Commit: aa0142aed933808f391bc1a64cb9dc7eff06784d Parents: 7622c57 57c6705 Author: sboikov <[email protected]> Authored: Thu May 11 12:13:48 2017 +0300 Committer: sboikov <[email protected]> Committed: Thu May 11 12:13:48 2017 +0300 ---------------------------------------------------------------------- .../store/cassandra/CassandraCacheStore.java | 16 +- .../session/LoadCacheCustomQueryWorker.java | 26 +- .../ignite/tests/IgnitePersistentStoreTest.java | 30 +- .../tests/persistence/blob/ignite-config.xml | 4 +- .../tests/persistence/pojo/ignite-config.xml | 4 +- .../persistence/primitive/ignite-config.xml | 4 +- .../primitive/ignite-remote-server-config.xml | 4 +- .../processors/rest/TestMemcacheClient.java | 2 +- .../org/apache/ignite/IgniteJdbcDriver.java | 69 +-- .../apache/ignite/IgniteSystemProperties.java | 3 + .../store/jdbc/CacheAbstractJdbcStore.java | 71 ++- .../internal/jdbc/JdbcDriverPropertyInfo.java | 37 ++ .../affinity/GridAffinityProcessor.java | 3 +- .../cache/DynamicCacheChangeBatch.java | 17 + .../processors/cache/GridCacheContext.java | 7 + .../processors/cache/GridCacheEntryInfo.java | 68 +-- .../processors/cache/GridCacheProcessor.java | 12 + .../processors/cache/IgniteCacheProxy.java | 41 +- .../cache/database/MemoryMetricsImpl.java | 8 +- .../GridDhtPartitionSupplyMessage.java | 30 +- .../internal/processors/odbc/IgniteTypes.java | 69 --- .../processors/odbc/OdbcBufferedParser.java | 81 --- .../processors/odbc/OdbcColumnMeta.java | 110 ---- .../processors/odbc/OdbcHandshakeRequest.java | 83 --- .../processors/odbc/OdbcHandshakeResult.java | 73 --- .../processors/odbc/OdbcMessageParser.java | 320 ---------- .../processors/odbc/OdbcNioListener.java | 153 +++-- .../processors/odbc/OdbcNioServerBuffer.java | 114 ---- .../internal/processors/odbc/OdbcProcessor.java | 2 +- .../processors/odbc/OdbcProtocolVersion.java | 125 ---- .../processors/odbc/OdbcQueryCloseRequest.java | 49 -- .../processors/odbc/OdbcQueryCloseResult.java | 40 -- .../odbc/OdbcQueryExecuteRequest.java | 78 --- .../processors/odbc/OdbcQueryExecuteResult.java | 54 -- .../processors/odbc/OdbcQueryFetchRequest.java | 61 -- .../processors/odbc/OdbcQueryFetchResult.java | 66 -- .../odbc/OdbcQueryGetColumnsMetaRequest.java | 4 +- .../odbc/OdbcQueryGetColumnsMetaResult.java | 6 +- .../odbc/OdbcQueryGetParamsMetaRequest.java | 4 +- .../odbc/OdbcQueryGetTablesMetaRequest.java | 4 +- .../internal/processors/odbc/OdbcRequest.java | 61 -- .../processors/odbc/OdbcRequestHandler.java | 532 ----------------- .../internal/processors/odbc/OdbcResponse.java | 96 --- .../internal/processors/odbc/OdbcUtils.java | 30 +- .../odbc/SqlListenerBufferedParser.java | 81 +++ .../processors/odbc/SqlListenerColumnMeta.java | 110 ++++ .../odbc/SqlListenerConnectionContext.java | 56 ++ .../processors/odbc/SqlListenerDataTypes.java | 69 +++ .../odbc/SqlListenerMessageParser.java | 39 ++ .../odbc/SqlListenerNioServerBuffer.java | 114 ++++ .../odbc/SqlListenerProtocolVersion.java | 116 ++++ .../odbc/SqlListenerQueryCloseRequest.java | 49 ++ .../odbc/SqlListenerQueryCloseResult.java | 40 ++ .../odbc/SqlListenerQueryExecuteRequest.java | 78 +++ .../odbc/SqlListenerQueryExecuteResult.java | 54 ++ .../odbc/SqlListenerQueryFetchRequest.java | 61 ++ .../odbc/SqlListenerQueryFetchResult.java | 66 ++ .../processors/odbc/SqlListenerRequest.java | 78 +++ .../odbc/SqlListenerRequestHandler.java | 31 + .../processors/odbc/SqlListenerResponse.java | 96 +++ .../odbc/escape/OdbcEscapeParseResult.java | 73 --- .../processors/odbc/escape/OdbcEscapeToken.java | 61 -- .../processors/odbc/escape/OdbcEscapeType.java | 112 ---- .../processors/odbc/escape/OdbcEscapeUtils.java | 440 -------------- .../processors/odbc/odbc/OdbcMessageParser.java | 279 +++++++++ .../odbc/odbc/OdbcRequestHandler.java | 513 ++++++++++++++++ .../odbc/odbc/escape/OdbcEscapeParseResult.java | 73 +++ .../odbc/odbc/escape/OdbcEscapeToken.java | 61 ++ .../odbc/odbc/escape/OdbcEscapeType.java | 112 ++++ .../odbc/odbc/escape/OdbcEscapeUtils.java | 441 ++++++++++++++ .../processors/query/GridQueryIndexing.java | 36 +- .../processors/query/GridQueryProcessor.java | 132 ++-- .../ignite/spi/discovery/tcp/ClientImpl.java | 21 +- .../ignite/spi/discovery/tcp/ServerImpl.java | 2 + .../spi/discovery/tcp/TcpDiscoverySpi.java | 7 + .../resources/META-INF/classnames.properties | 2 +- .../CacheJdbcPojoStoreAbstractSelfTest.java | 49 ++ .../cache/distributed/CacheStartOnJoinTest.java | 250 ++++++++ .../IgniteCacheExpiryPolicyAbstractTest.java | 15 +- .../odbc/OdbcEscapeSequenceSelfTest.java | 2 +- .../ignite/messaging/GridMessagingSelfTest.java | 4 +- .../testframework/junits/GridAbstractTest.java | 12 + .../testsuites/IgniteCacheTestSuite4.java | 2 + modules/docker/2.0.0/Dockerfile | 46 ++ modules/docker/2.0.0/run.sh | 51 ++ modules/docker/Dockerfile | 2 +- .../cache/hibernate/HibernateKeyWrapper.java | 7 + .../HibernateL2CacheStrategySelfTest.java | 597 +++++++++++++++++++ .../testsuites/IgniteHibernateTestSuite.java | 2 + .../hibernate/HibernateL2CacheSelfTest.java | 2 +- .../HibernateL2CacheStrategySelfTest.java | 569 ++++++++++++++++++ .../testsuites/IgniteHibernate5TestSuite.java | 2 + .../query/h2/DmlStatementsProcessor.java | 11 +- .../processors/query/h2/IgniteH2Indexing.java | 20 +- .../query/h2/sql/GridSqlQuerySplitter.java | 7 +- .../cache/GridCacheCrossCacheQuerySelfTest.java | 12 +- ...niteCacheAbstractInsertSqlQuerySelfTest.java | 3 +- .../IgniteCacheInsertSqlQuerySelfTest.java | 22 + .../query/IgniteSqlSplitterSelfTest.java | 30 + .../cpp/odbc-test/project/vs/odbc-test.vcxproj | 1 + .../project/vs/odbc-test.vcxproj.filters | 4 + .../cpp/odbc-test/src/configuration_test.cpp | 40 +- .../cpp/odbc-test/src/queries_test.cpp | 62 +- .../cpp/odbc/include/ignite/odbc/message.h | 45 +- .../odbc/include/ignite/odbc/protocol_version.h | 103 ++-- .../src/system/ui/dsn_configuration_window.cpp | 42 +- modules/platforms/cpp/odbc/src/connection.cpp | 35 +- .../platforms/cpp/odbc/src/protocol_version.cpp | 138 +++-- .../Apache.Ignite.Core.Tests.csproj | 1 + .../Binary/BinaryDynamicRegistrationTest.cs | 40 ++ .../Binary/BinaryNameMapperTest.cs | 7 + .../Binary/TypeNameParserTest.cs | 93 ++- .../Binary/TypeResolverTest.cs | 104 ++++ .../Cache/Query/CacheQueriesTest.cs | 5 +- .../Apache.Ignite.Core.Tests/DeploymentTest.cs | 1 + .../Examples/ExamplesTest.cs | 5 +- .../Binary/BinaryBasicNameMapper.cs | 6 +- .../Impl/Binary/BinaryProcessor.cs | 16 +- .../Impl/Binary/BinaryReader.cs | 6 +- .../Impl/Binary/Marshaller.cs | 18 +- .../Impl/Binary/TypeNameParser.cs | 31 +- .../Impl/Binary/TypeResolver.cs | 115 +++- .../Impl/Memory/PlatformMemoryStream.cs | 2 +- modules/web-console/backend/app/mongo.js | 1 + .../generator/ConfigurationGenerator.js | 3 +- .../states/configuration/domains/general.pug | 4 +- .../states/configuration/domains/query.pug | 2 + .../frontend/controllers/domains-controller.js | 26 +- .../views/configuration/domains-import.tpl.pug | 11 +- 129 files changed, 5441 insertions(+), 3417 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/aa0142ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java ---------------------------------------------------------------------- diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java index 3c65326,66e780f..e72f214 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java @@@ -39,9 -37,19 +39,12 @@@ public class DynamicCacheChangeBatch im @GridToStringInclude private Collection<DynamicCacheChangeRequest> reqs; - /** Client nodes map. Used in discovery data exchange. */ - @GridToStringInclude - private Map<String, Map<UUID, Boolean>> clientNodes; - - /** Custom message ID. */ - private IgniteUuid id = IgniteUuid.randomUuid(); - - /** */ - private boolean clientReconnect; + /** Cache updates to be executed on exchange. */ + private transient ExchangeActions exchangeActions; + /** */ + private boolean startCaches; + /** * @param reqs Requests. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/aa0142ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/aa0142ae/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java ---------------------------------------------------------------------- diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 2f06d0d,87aaee0..b446b32 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@@ -2035,33 -2137,322 +2040,40 @@@ public class GridCacheProcessor extend /** {@inheritDoc} */ @Override public void collectGridNodeData(DiscoveryDataBag dataBag) { - dataBag.addNodeSpecificData(CACHE_PROC.ordinal(), getDiscoveryData(dataBag.joiningNodeId())); + cachesInfo.collectGridNodeData(dataBag); } + /** - * @param joiningNodeId Joining node id. ++ * @return {@code True} if need locally start all existing caches on client node start. + */ - private Serializable getDiscoveryData(UUID joiningNodeId) { - boolean reconnect = ctx.localNodeId().equals(joiningNodeId) && cachesOnDisconnect != null; - - // Collect dynamically started caches to a single object. - Collection<DynamicCacheChangeRequest> reqs; ++ private boolean startAllCachesOnClientStart() { ++ return START_CLIENT_CACHES && ctx.clientNode(); ++ } + - Map<String, Map<UUID, Boolean>> clientNodesMap; + /** {@inheritDoc} */ + @Override public void onJoiningNodeDataReceived(JoiningNodeDiscoveryData data) { + cachesInfo.onJoiningNodeDataReceived(data); + } - if (reconnect) { - reqs = new ArrayList<>(caches.size() + 1); + /** {@inheritDoc} */ + @Override public void onGridDataReceived(GridDiscoveryData data) { + cachesInfo.onGridDataReceived(data); + } - clientNodesMap = U.newHashMap(caches.size()); + /** + * Dynamically starts cache using template configuration. + * + * @param cacheName Cache name. + * @return Future that will be completed when cache is deployed. + */ + public IgniteInternalFuture<?> createFromTemplate(String cacheName) { + try { + CacheConfiguration cfg = createConfigFromTemplate(cacheName); - collectDataOnReconnectingNode(reqs, clientNodesMap, joiningNodeId); + return dynamicStartCache(cfg, cacheName, null, true, true, true); } - else { - reqs = new ArrayList<>(registeredCaches.size() + registeredTemplates.size() + 1); - - clientNodesMap = ctx.discovery().clientNodesMap(); - - collectDataOnGridNode(reqs); - } - - DynamicCacheChangeBatch batch = new DynamicCacheChangeBatch(reqs); - - batch.clientNodes(clientNodesMap); - - batch.clientReconnect(reconnect); - - if (ctx.localNodeId().equals(joiningNodeId)) - batch.startCaches(startAllCachesOnClientStart()); - - // Reset random batch ID so that serialized batches with the same descriptors will be exactly the same. - batch.id(null); - - return batch; - } - - /** - * @param reqs requests. - */ - private void collectDataOnGridNode(Collection<DynamicCacheChangeRequest> reqs) { - for (DynamicCacheDescriptor desc : cacheDescriptors()) { - // RequestId must be null because on different node will be different byte [] and - // we get duplicate discovery data, for more details see - // TcpDiscoveryNodeAddedMessage#addDiscoveryData. - DynamicCacheChangeRequest req = new DynamicCacheChangeRequest(null, desc.cacheConfiguration().getName(), - null); - - req.startCacheConfiguration(desc.cacheConfiguration()); - req.cacheType(desc.cacheType()); - req.deploymentId(desc.deploymentId()); - req.receivedFrom(desc.receivedFrom()); - req.schema(desc.schema()); - - reqs.add(req); - } - - for (DynamicCacheDescriptor desc : registeredTemplates.values()) { - // RequestId must be null because on different node will be different byte [] and - // we get duplicate discovery data, for more details see - // TcpDiscoveryNodeAddedMessage#addDiscoveryData. - DynamicCacheChangeRequest req = new DynamicCacheChangeRequest(null, desc.cacheConfiguration().getName(), - null); - - req.startCacheConfiguration(desc.cacheConfiguration()); - req.schema(desc.schema()); - - req.template(true); - - reqs.add(req); - } - } - - /** - * @param reqs requests. - * @param clientNodesMap Client nodes map. - * @param nodeId Node id. - */ - private void collectDataOnReconnectingNode( - Collection<DynamicCacheChangeRequest> reqs, - Map<String, Map<UUID, Boolean>> clientNodesMap, - UUID nodeId - ) { - for (GridCacheAdapter<?, ?> cache : caches.values()) { - DynamicCacheDescriptor desc = cachesOnDisconnect.get(cache.name()); - - if (desc == null) - continue; - - DynamicCacheChangeRequest req = new DynamicCacheChangeRequest(null, cache.name(), null); - - req.startCacheConfiguration(desc.cacheConfiguration()); - req.cacheType(desc.cacheType()); - req.deploymentId(desc.deploymentId()); - req.receivedFrom(desc.receivedFrom()); - req.schema(desc.schema()); - - reqs.add(req); - - Boolean nearEnabled = cache.isNear(); - - Map<UUID, Boolean> map = U.newHashMap(1); - - map.put(nodeId, nearEnabled); - - clientNodesMap.put(cache.name(), map); - } - } - - /** - * @return {@code True} if need locally start all existing caches on client node start. - */ - private boolean startAllCachesOnClientStart() { - return START_CLIENT_CACHES && ctx.clientNode(); - } - - /** {@inheritDoc} */ - @Override public void onJoiningNodeDataReceived(JoiningNodeDiscoveryData data) { - if (data.hasJoiningNodeData()) { - Serializable joiningNodeData = data.joiningNodeData(); - if (joiningNodeData instanceof DynamicCacheChangeBatch) - onDiscoDataReceived( - data.joiningNodeId(), - data.joiningNodeId(), - (DynamicCacheChangeBatch) joiningNodeData, true); - } - } - - /** {@inheritDoc} */ - @Override public void onGridDataReceived(GridDiscoveryData data) { - Map<UUID, Serializable> nodeSpecData = data.nodeSpecificData(); - - if (nodeSpecData != null) { - for (Map.Entry<UUID, Serializable> e : nodeSpecData.entrySet()) { - if (e.getValue() != null && e.getValue() instanceof DynamicCacheChangeBatch) { - DynamicCacheChangeBatch batch = (DynamicCacheChangeBatch) e.getValue(); - - onDiscoDataReceived(data.joiningNodeId(), e.getKey(), batch, false); - } - } - } - } - - /** - * @param joiningNodeId Joining node id. - * @param rmtNodeId Rmt node id. - * @param batch Batch. - * @param join Whether this is data from joining node. - */ - private void onDiscoDataReceived(UUID joiningNodeId, UUID rmtNodeId, DynamicCacheChangeBatch batch, boolean join) { - if (batch.clientReconnect()) { - if (ctx.clientDisconnected()) { - if (clientReconnectReqs == null) - clientReconnectReqs = new LinkedHashMap<>(); - - clientReconnectReqs.put(joiningNodeId, batch); - - return; - } - - processClientReconnectData(joiningNodeId, batch); - } - else { - for (DynamicCacheChangeRequest req : batch.requests()) { - initReceivedCacheConfiguration(req); - - if (req.template()) { - CacheConfiguration ccfg = req.startCacheConfiguration(); - - assert ccfg != null : req; - - DynamicCacheDescriptor existing = registeredTemplates.get(req.cacheName()); - - if (existing == null) { - DynamicCacheDescriptor desc = new DynamicCacheDescriptor( - ctx, - ccfg, - req.cacheType(), - true, - req.deploymentId(), - req.schema()); - - registeredTemplates.put(req.cacheName(), desc); - } - - continue; - } - - DynamicCacheDescriptor existing = cacheDescriptor(req.cacheName()); - - if (req.start() && !req.clientStartOnly()) { - CacheConfiguration ccfg = req.startCacheConfiguration(); - - if (existing != null) { - if (joiningNodeId.equals(ctx.localNodeId())) { - existing.receivedFrom(req.receivedFrom()); - existing.deploymentId(req.deploymentId()); - } - - if (existing.locallyConfigured()) { - existing.addRemoteConfiguration(rmtNodeId, req.startCacheConfiguration()); - - if (!join) - // Overwrite existing with remote. - existing.schema(req.schema()); - - ctx.discovery().setCacheFilter( - req.cacheName(), - ccfg.getNodeFilter(), - ccfg.getNearConfiguration() != null, - ccfg.getCacheMode()); - } - } - else { - assert req.cacheType() != null : req; - - DynamicCacheDescriptor desc = new DynamicCacheDescriptor( - ctx, - ccfg, - req.cacheType(), - false, - req.deploymentId(), - req.schema()); - - // Received statically configured cache. - if (req.initiatingNodeId() == null) - desc.staticallyConfigured(true); - - if (joiningNodeId.equals(ctx.localNodeId())) - desc.receivedOnDiscovery(true); - - desc.receivedFrom(req.receivedFrom()); - - DynamicCacheDescriptor old = cacheDescriptor(req.cacheName(), desc); - - assert old == null : old; - - ctx.discovery().setCacheFilter( - req.cacheName(), - ccfg.getNodeFilter(), - ccfg.getNearConfiguration() != null, - ccfg.getCacheMode()); - } - } - } - - if (!F.isEmpty(batch.clientNodes())) { - for (Map.Entry<String, Map<UUID, Boolean>> entry : batch.clientNodes().entrySet()) { - String cacheName = entry.getKey(); - - for (Map.Entry<UUID, Boolean> tup : entry.getValue().entrySet()) - ctx.discovery().addClientNode(cacheName, tup.getKey(), tup.getValue()); - } - } - - if (batch.startCaches()) { - for (Map.Entry<String, DynamicCacheDescriptor> entry : registeredCaches.entrySet()) - ctx.discovery().addClientNode(entry.getKey(), joiningNodeId, false); - } - } - } - - /** - * @param clientNodeId Client node ID. - * @param batch Cache change batch. - */ - private void processClientReconnectData(UUID clientNodeId, DynamicCacheChangeBatch batch) { - assert batch.clientReconnect() : batch; - - for (DynamicCacheChangeRequest req : batch.requests()) { - assert !req.template() : req; - - initReceivedCacheConfiguration(req); - - String name = req.cacheName(); - - boolean sysCache = CU.isUtilityCache(name) || CU.isAtomicsCache(name); - - if (!sysCache) { - DynamicCacheDescriptor desc = cacheDescriptor(req.cacheName()); - - if (desc != null && desc.deploymentId().equals(req.deploymentId())) { - Map<UUID, Boolean> nodes = batch.clientNodes().get(name); - - assert nodes != null : req; - assert nodes.containsKey(clientNodeId) : nodes; - - ctx.discovery().addClientNode(req.cacheName(), clientNodeId, nodes.get(clientNodeId)); - } - } - else - ctx.discovery().addClientNode(req.cacheName(), clientNodeId, false); - } - } - - /** - * Dynamically starts cache using template configuration. - * - * @param cacheName Cache name. - * @return Future that will be completed when cache is deployed. - */ - public IgniteInternalFuture<?> createFromTemplate(String cacheName) { - try { - CacheConfiguration cfg = createConfigFromTemplate(cacheName); - - return dynamicStartCache(cfg, cacheName, null, true, true, true); - } - catch (IgniteCheckedException e) { - throw U.convertException(e); + catch (IgniteCheckedException e) { + throw U.convertException(e); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/aa0142ae/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/aa0142ae/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java ----------------------------------------------------------------------
