Repository: usergrid Updated Branches: refs/heads/master 40e1c0660 -> 2578e938a
Give CoreSchemaManager in tests the ability to force check if keyspaces are created while normal runtime classes can rely on the driver's cached metadata. Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2578e938 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2578e938 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2578e938 Branch: refs/heads/master Commit: 2578e938a2a3cecd3c90d1d946a7ff50669fc320 Parents: 40e1c06 Author: Michael Russo <mru...@apigee.com> Authored: Sat Nov 19 09:03:48 2016 -0800 Committer: Michael Russo <mru...@apigee.com> Committed: Sat Nov 19 09:03:48 2016 -0800 ---------------------------------------------------------------------- .../corepersistence/CpEntityManagerFactory.java | 4 +- .../usergrid/corepersistence/CpSetup.java | 9 ++-- .../usergrid/persistence/cassandra/Setup.java | 3 +- .../usergrid/persistence/CoreSchemaManager.java | 4 +- .../core/datastax/DataStaxCluster.java | 4 +- .../core/datastax/impl/DataStaxClusterImpl.java | 47 +++++++++++--------- .../core/migration/schema/MigrationManager.java | 3 +- .../migration/schema/MigrationManagerImpl.java | 6 +-- .../core/guice/MigrationManagerRule.java | 4 +- .../data/MigrationInfoSerializationTest.java | 2 +- .../graph/GraphManagerShardConsistencyIT.java | 2 +- .../apache/usergrid/persistence/qakka/App.java | 4 +- .../persistence/qakka/AbstractTest.java | 2 +- .../usergrid/rest/system/MigrateResource.java | 2 +- .../org/apache/usergrid/tools/ToolBase.java | 2 +- 15 files changed, 52 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 9caa67e..cec7258 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@ -711,7 +711,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application @Override public void setup() throws Exception { - getSetup().initSchema(); + getSetup().initSchema(false); lockManager.setup(); } @@ -720,7 +720,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application public void bootstrap() throws Exception { // Always make sure the database schema is initialized - getSetup().initSchema(); + getSetup().initSchema(false); // Roll the new 2.x Migration classes to the latest version supported getSetup().runDataMigration(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java index 28b1ccc..d7c63a3 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java @@ -75,13 +75,13 @@ public class CpSetup implements Setup { @Override - public void initSchema() throws Exception { + public void initSchema(boolean forceCheckKeyspaces) throws Exception { // Initialize the management app index in Elasticsearch this.emf.initializeManagementIndex(); // Create the schema (including keyspace) in Cassandra - setupSchema(); + setupSchema(forceCheckKeyspaces); setupLegacySchema(); } @@ -135,13 +135,14 @@ public class CpSetup implements Setup { /** * Initialize schema from the new 2.x Migration classes which contain schema individually * + * @param forceCheckKeyspaces */ - private void setupSchema() throws Exception { + private void setupSchema(boolean forceCheckKeyspaces) throws Exception { MigrationManager m = injector.getInstance( MigrationManager.class ); try { - m.migrate(); + m.migrate(forceCheckKeyspaces); } catch ( MigrationException ex ) { throw new RuntimeException( "Error migrating Core Persistence", ex ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/Setup.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/Setup.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/Setup.java index 697cd21..c79ff7f 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/Setup.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/Setup.java @@ -21,8 +21,9 @@ public interface Setup { /** * Initialize all configuration for the system setup. Creates keyspaces and elasticsearch indexes * @throws Exception + * @param forceCheckKeyspaces */ - void initSchema() throws Exception; + void initSchema(boolean forceCheckKeyspaces) throws Exception; /** http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java index 087fa37..8054aa5 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java @@ -18,8 +18,6 @@ package org.apache.usergrid.persistence; import org.apache.usergrid.locking.LockManager; -import org.apache.usergrid.persistence.queue.LegacyQueueManager; -import org.apache.usergrid.persistence.queue.impl.QakkaQueueManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +51,7 @@ public class CoreSchemaManager implements SchemaManager { @Override public void create() { try { - setup.initSchema(); + setup.initSchema(true); lockManager.setup(); } catch ( Exception ex ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java index 008e8e1..6c0c9e2 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java @@ -32,9 +32,9 @@ public interface DataStaxCluster { Session getApplicationLocalSession(); - void createApplicationKeyspace() throws Exception; + void createApplicationKeyspace(boolean forceCheck) throws Exception; - void createApplicationLocalKeyspace() throws Exception; + void createApplicationLocalKeyspace(boolean forceCheck) throws Exception; void waitForSchemaAgreement(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java index 9e942d0..2377c67 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java @@ -22,7 +22,6 @@ import com.datastax.driver.core.*; import com.datastax.driver.core.policies.DCAwareRoundRobinPolicy; import com.datastax.driver.core.policies.LoadBalancingPolicy; import com.datastax.driver.core.policies.Policies; -import com.datastax.driver.core.policies.ReconnectionPolicy; import com.google.inject.Inject; import com.google.inject.Singleton; import org.apache.usergrid.persistence.core.CassandraConfig; @@ -56,7 +55,7 @@ public class DataStaxClusterImpl implements DataStaxCluster { cluster.getConfiguration().getPoolingOptions().getPoolTimeoutMillis() / 1000); // always initialize the keyspaces - this.createApplicationKeyspace(); + this.createApplicationKeyspace(false); } @Override @@ -106,19 +105,23 @@ public class DataStaxClusterImpl implements DataStaxCluster { /** * Execute CQL that will create the keyspace if it doesn't exist and alter it if it does. * @throws Exception + * @param forceCheck */ @Override - public synchronized void createApplicationKeyspace() throws Exception { + public synchronized void createApplicationKeyspace(boolean forceCheck) throws Exception { - // using the datastax client seems to have stale data ( as seen in tests that drop keyspaces ) - //boolean exists = getClusterSession().getCluster().getMetadata() - // .getKeyspace(CQLUtils.quote( cassandraConfig.getApplicationKeyspace())) != null; - - Row row = getClusterSession() - .execute("select * from system.schema_keyspaces where keyspace_name = '"+cassandraConfig.getApplicationKeyspace()+"'") - .one(); + boolean exists; + if(!forceCheck) { + // this gets info from client's metadata + exists = getClusterSession().getCluster().getMetadata() + .getKeyspace(CQLUtils.quote(cassandraConfig.getApplicationKeyspace())) != null; + }else{ + exists = getClusterSession() + .execute("select * from system.schema_keyspaces where keyspace_name = '"+cassandraConfig.getApplicationKeyspace()+"'") + .one() != null; + } - if(row != null){ + if(exists){ logger.info("Not creating keyspace {}, it already exists.", cassandraConfig.getApplicationKeyspace()); return; } @@ -141,19 +144,23 @@ public class DataStaxClusterImpl implements DataStaxCluster { /** * Execute CQL that will create the keyspace if it doesn't exist and alter it if it does. * @throws Exception + * @param forceCheck */ @Override - public synchronized void createApplicationLocalKeyspace() throws Exception { + public synchronized void createApplicationLocalKeyspace(boolean forceCheck) throws Exception { - // using the datastax client seems to have stale data ( as seen in tests that drop keyspaces ) - //boolean exists = getClusterSession().getCluster().getMetadata() - // .getKeyspace(CQLUtils.quote( cassandraConfig.getApplicationLocalKeyspace())) != null; - - Row row = getClusterSession() - .execute("select * from system.schema_keyspaces where keyspace_name = '"+cassandraConfig.getApplicationLocalKeyspace()+"'") - .one(); + boolean exists; + if(!forceCheck) { + // this gets info from client's metadata + exists = getClusterSession().getCluster().getMetadata() + .getKeyspace(CQLUtils.quote(cassandraConfig.getApplicationLocalKeyspace())) != null; + }else{ + exists = getClusterSession() + .execute("select * from system.schema_keyspaces where keyspace_name = '"+cassandraConfig.getApplicationLocalKeyspace()+"'") + .one() != null; + } - if (row != null) { + if (exists) { logger.info("Not creating keyspace {}, it already exists.", cassandraConfig.getApplicationLocalKeyspace()); return; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManager.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManager.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManager.java index df84247..8b2dd41 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManager.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManager.java @@ -29,6 +29,7 @@ public interface MigrationManager { /** * Perform any migration necessary in the application. Will only create keyspaces and column families if they do * not exist + * @param forceCheckKeyspaces */ - public void migrate() throws MigrationException; + void migrate(boolean forceCheckKeyspaces) throws MigrationException; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java index 2205e57..b648e66 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java @@ -69,13 +69,13 @@ public class MigrationManagerImpl implements MigrationManager { @Override - public void migrate() throws MigrationException { + public void migrate(boolean forceCheckKeyspaces) throws MigrationException { try { - dataStaxCluster.createApplicationKeyspace(); + dataStaxCluster.createApplicationKeyspace(forceCheckKeyspaces); - dataStaxCluster.createApplicationLocalKeyspace(); + dataStaxCluster.createApplicationLocalKeyspace(forceCheckKeyspaces); for ( Migration migration : migrations ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java index 4e5ac49..92149c1 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java @@ -45,7 +45,7 @@ public class MigrationManagerRule extends ExternalResource { this.migrationManager = migrationManager; try { - this.migrationManager.migrate(); + this.migrationManager.migrate(false); } catch ( MigrationException e ) { throw new RuntimeException(e); } @@ -60,7 +60,7 @@ public class MigrationManagerRule extends ExternalResource { protected void before() throws MigrationException { logger.info( "Starting migration" ); - migrationManager.migrate(); + migrationManager.migrate(false); logger.info("Migrating data"); http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java index 1b6dd3f..f3a884d 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java @@ -71,7 +71,7 @@ public class MigrationInfoSerializationTest { //drop the column family, then run setup keyspace.dropColumnFamily( MigrationInfoSerializationImpl.CF_MIGRATION_INFO.getName() ); - migrationManager.migrate(); + migrationManager.migrate(false); //test getting nothing works final String emptyStatus = migrationInfoSerialization.getStatusMessage(migrationModule); http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java index 6e6abd8..8cdd9a1 100644 --- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java +++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java @@ -414,7 +414,7 @@ public class GraphManagerShardConsistencyIT { final MigrationManager migrationManager = getInstance( injectors, MigrationManager.class ); - migrationManager.migrate(); + migrationManager.migrate(false); return injectors; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/App.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/App.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/App.java index 35fdb20..29197a2 100644 --- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/App.java +++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/App.java @@ -21,13 +21,11 @@ package org.apache.usergrid.persistence.qakka; import com.codahale.metrics.MetricRegistry; import com.google.inject.Inject; -import com.google.inject.Injector; import com.google.inject.Singleton; import org.apache.usergrid.persistence.actorsystem.ActorSystemFig; import org.apache.usergrid.persistence.actorsystem.ActorSystemManager; import org.apache.usergrid.persistence.core.migration.schema.MigrationException; import org.apache.usergrid.persistence.core.migration.schema.MigrationManager; -import org.apache.usergrid.persistence.qakka.core.Queue; import org.apache.usergrid.persistence.qakka.distributed.DistributedQueueService; import org.apache.usergrid.persistence.qakka.distributed.impl.QueueActorRouterProducer; import org.apache.usergrid.persistence.qakka.distributed.impl.QueueSenderRouterProducer; @@ -69,7 +67,7 @@ public class App implements MetricsService { if ( qakkaFig.getStandalone() ) { try { - migrationManager.migrate(); + migrationManager.migrate(false); } catch (MigrationException e) { throw new QakkaRuntimeException( "Error running migration", e ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/AbstractTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/AbstractTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/AbstractTest.java index 83939b2..788ca70 100644 --- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/AbstractTest.java +++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/AbstractTest.java @@ -43,7 +43,7 @@ public abstract class AbstractTest { if ( !migrated.getAndSet( true ) ) { MigrationManager migrationManager = getInjector().getInstance( MigrationManager.class ); try { - migrationManager.migrate(); + migrationManager.migrate(false); } catch (MigrationException e) { logger.error("Error in migration", e); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java index 5bfeac9..b0a71ab 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java @@ -86,7 +86,7 @@ public class MigrateResource extends AbstractContextResource { logger.info( "Migrating Schema" ); try { - getMigrationManager().migrate(); + getMigrationManager().migrate(false); } catch ( Exception e ) { logger.error( "Unable to migrate data", e ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/2578e938/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java index c33c5f5..87740b7 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java @@ -205,7 +205,7 @@ public abstract class ToolBase { Setup setup = ( (CpEntityManagerFactory) emf ).getSetup(); logger.info( "Setting up Usergrid schema" ); - setup.initSchema(); + setup.initSchema(false); setup.initMgmtApp(); logger.info( "Usergrid schema setup" );
