http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java index f39ef9b..7a925b0 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java @@ -17,20 +17,20 @@ package org.apache.usergrid.tools; -import com.sun.org.apache.bcel.internal.generic.DUP; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.io.filefilter.PrefixFileFilter; +import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.management.OrganizationInfo; import org.apache.usergrid.management.UserInfo; import org.apache.usergrid.persistence.EntityManager; import org.apache.usergrid.persistence.EntityRef; -import org.apache.usergrid.persistence.Identifier; import org.apache.usergrid.persistence.SimpleEntityRef; import org.apache.usergrid.persistence.entities.User; import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException; +import org.apache.usergrid.persistence.index.query.Identifier; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonParser; import org.codehaus.jackson.JsonToken; @@ -39,7 +39,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -47,18 +50,16 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE; import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID; -import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID; - /** * Import Admin Users and metadata including organizations and passwords. - * - * Usage Example: - * + * + * Usage Example: + * * java -Xmx8000m -Dlog4j.configuration=file:/home/me/log4j.properties -classpath . \ * -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 \ - * -host casshost -inputDir=/home/me/import-data - * + * -host casshost -inputDir=/home/me/import-data + * * If you want to provide any property overrides, put properties file named usergrid-custom-tools.properties * in the same directory where you run the above command. For example, you might want to set the Cassandra * client threads and import to a specific set of keyspaces: @@ -97,8 +98,8 @@ public class ImportAdmins extends ToolBase { AtomicInteger writeEmptyCount = new AtomicInteger( 0 ); AtomicInteger auditEmptyCount = new AtomicInteger( 0 ); AtomicInteger metadataEmptyCount = new AtomicInteger( 0 ); - - + + static class DuplicateUser { String email; String username; @@ -110,7 +111,7 @@ public class ImportAdmins extends ToolBase { } } } - + @Override @@ -269,7 +270,7 @@ public class ImportAdmins extends ToolBase { long timeRemaining = (long) ( sizeLast / (rateAverageSum / iterations) ); logger.info("++PROGRESS ({}): sizeLast={} nowSize={} processed={} rateLast={}/s rateAvg={}/s timeRemaining={}s", - new Object[] { + new Object[] { identifier, sizeLast, sizeNow, processed, rateLast, (rateAverageSum / iterations), timeRemaining } ); } @@ -355,7 +356,7 @@ public class ImportAdmins extends ToolBase { workerThread.start(); metadataWorkerThreadMap.put(worker, workerThread); } - + logger.info( "Started {} metadata workers", writeThreadCount ); } @@ -363,7 +364,7 @@ public class ImportAdmins extends ToolBase { @SuppressWarnings("unchecked") private void importMetadata(String fileName, int writeThreads) throws Exception { - EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID); + EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID); File metadataFile = new File(importDir, fileName); @@ -401,10 +402,10 @@ public class ImportAdmins extends ToolBase { try { EntityRef entityRef = new SimpleEntityRef( "user", UUID.fromString( entityOwnerId ) ); Map<String, Object> metadata = (Map<String, Object>) jp.readValueAs( Map.class ); - + workQueue.put( new ImportMetadataTask( entityRef, metadata ) ); logger.debug( "Put user {} in metadata queue", entityRef.getUuid() ); - + } catch ( Exception e ) { logger.debug( "Error with user {}, not putting in metadata queue", entityOwnerId ); } @@ -426,7 +427,7 @@ public class ImportAdmins extends ToolBase { EntityManager em, EntityRef entityRef, Map<String, Object> metadata) throws Exception { DuplicateUser dup = dupsByDupUuid.get( entityRef.getUuid() ); - + if ( dup == null ) { // not a duplicate User user = em.get( entityRef, User.class ); @@ -476,7 +477,7 @@ public class ImportAdmins extends ToolBase { } } } - + Map<String, Object> dictionariesMap = (Map<String, Object>) metadata.get("dictionaries"); if (dictionariesMap != null && !dictionariesMap.isEmpty()) { for (String name : dictionariesMap.keySet()) { @@ -501,17 +502,17 @@ public class ImportAdmins extends ToolBase { } else { logger.warn("User {} has no dictionaries", entityRef.getUuid() ); } - + } else { // this is a duplicate user, so merge orgs logger.info("Processing duplicate username={} email={}", dup.email, dup.username ); - - Identifier identifier = dup.email != null ? + + Identifier identifier = dup.email != null ? Identifier.fromEmail( dup.email ) : Identifier.from( dup.username ); User originalUser = em.get( em.getUserByIdentifier(identifier), User.class ); // get map of original user's orgs - + UserInfo originalUserInfo = managementService.getAdminUserByEmail( originalUser.getEmail() ); Map<String, Object> originalUserOrgData = managementService.getAdminUserOrganizationData( originalUser.getUuid() ); @@ -530,7 +531,7 @@ public class ImportAdmins extends ToolBase { if (originalUserOrgs.get( orgName ) == null) { // original user does not have this org OrganizationInfo orgInfo = managementService.getOrganizationByUuid( orgUuid ); - + if (orgInfo == null) { // org does not exist yet, create it and add original user to it try { managementService.createOrganization( orgUuid, orgName, originalUserInfo, false ); @@ -539,9 +540,9 @@ public class ImportAdmins extends ToolBase { logger.debug( "Created new org {} for user {}:{}:{} from duplicate user {}:{}", new Object[]{ orgInfo.getName(), - originalUser.getUsername(), + originalUser.getUsername(), originalUser.getEmail(), - originalUser.getUuid(), + originalUser.getUuid(), dup.username, dup.email }); @@ -554,20 +555,20 @@ public class ImportAdmins extends ToolBase { logger.debug( "Added to org user {}:{}:{} from duplicate user {}:{}", new Object[]{ orgInfo.getName(), - originalUser.getUsername(), + originalUser.getUsername(), originalUser.getEmail(), - originalUser.getUuid(), + originalUser.getUuid(), dup.username, dup.email }); } catch (Exception e) { - logger.error( "Error Adding user {} to org {}", + logger.error( "Error Adding user {} to org {}", new Object[]{originalUserInfo.getEmail(), orgName} ); } } - + } // else original user already has this org - + } } } @@ -612,7 +613,7 @@ public class ImportAdmins extends ToolBase { public void run() { int count = 0; - EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID); + EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID); long durationSum = 0; @@ -647,7 +648,7 @@ public class ImportAdmins extends ToolBase { durationSum += duration; //logger.debug( "Audited {}th admin", userCount ); - + if ( count % 100 == 0 ) { logger.info( "Audited {}. Average Audit Rate: {}(ms)", count, durationSum / count ); } @@ -690,7 +691,7 @@ public class ImportAdmins extends ToolBase { public void run() { int count = 0; - EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID); + EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID); long durationSum = 0; @@ -705,20 +706,20 @@ public class ImportAdmins extends ToolBase { continue; } metadataEmptyCount.set( 0 ); - + long startTime = System.currentTimeMillis(); - + importEntityMetadata( em, task.entityRef, task.metadata ); - + long stopTime = System.currentTimeMillis(); long duration = stopTime - startTime; durationSum += duration; metadataCount.getAndIncrement(); count++; - + if ( count % 30 == 0 ) { logger.info( "Imported {} metadata of total {} expected. " + - "Average metadata Imported Rate: {}(ms)", + "Average metadata Imported Rate: {}(ms)", new Object[] { metadataCount.get(), userCount.get(), durationSum / count }); } @@ -752,7 +753,7 @@ public class ImportAdmins extends ToolBase { public void run() { int count = 0; - EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID); + EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID); long durationSum = 0; @@ -776,7 +777,7 @@ public class ImportAdmins extends ToolBase { try { long startTime = System.currentTimeMillis(); - + em.create(uuid, type, entityProps); logger.debug( "Imported admin user {}:{}:{}", @@ -787,20 +788,20 @@ public class ImportAdmins extends ToolBase { long stopTime = System.currentTimeMillis(); long duration = stopTime - startTime; durationSum += duration; - + count++; if (count % 30 == 0) { logger.info( "This worked has imported {} users of total {} imported so far. " + - "Average Creation Rate: {}ms", + "Average Creation Rate: {}ms", new Object[] { count, userCount.get(), durationSum / count }); } - + } catch (DuplicateUniquePropertyExistsException de) { String dupProperty = de.getPropertyName(); handleDuplicateAccount( em, dupProperty, entityProps ); continue; - + } catch (Exception e) { logger.error("Error", e); } @@ -811,19 +812,19 @@ public class ImportAdmins extends ToolBase { } } - + private void handleDuplicateAccount(EntityManager em, String dupProperty, Map<String, Object> entityProps ) { logger.info( "Processing duplicate user {}:{}:{} with duplicate {}", new Object[]{ - entityProps.get( "username" ), - entityProps.get( "email" ), - entityProps.get( "uuid" ), + entityProps.get( "username" ), + entityProps.get( "email" ), + entityProps.get( "uuid" ), dupProperty} ); - + UUID dupUuid = UUID.fromString( entityProps.get("uuid").toString() ); try { dupsByDupUuid.put( dupUuid, new DuplicateUser( dupProperty, entityProps ) ); - + } catch (Exception e) { logger.error("Error processing dup user {}:{}:{}", new Object[] {entityProps.get( "username" ), entityProps.get("email"), dupUuid});
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java b/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java deleted file mode 100644 index 5c35e85..0000000 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.tools; - - -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.UUID; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.usergrid.persistence.EntityManager; -import org.apache.usergrid.utils.UUIDUtils; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.usergrid.persistence.EntityManagerFactory; -import org.apache.usergrid.persistence.EntityRef; - - -/** - * Index rebuild utility for Usergrid. Can be used to rebuild the index for a specific - * application, a specific application's collection or for an entire Usergrid system. - */ -public class IndexRebuild extends ToolBase { - - private static final String APPLICATION_ARG = "app"; - - private static final String COLLECTION_ARG = "col"; - - private static final String ALL_ARG = "all"; - - private static final int PAGE_SIZE = 100; - - - private static final Logger logger = LoggerFactory.getLogger( IndexRebuild.class ); - - - @Override - @SuppressWarnings("static-access") - public Options createOptions() { - - Option hostOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ) - .withDescription( "Cassandra host" ).create( "host" ); - - Option esHostsOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ) - .withDescription( "ElasticSearch host" ).create( "eshost" ); - - Option esClusterOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ) - .withDescription( "ElasticSearch cluster name" ).create( "escluster" ); - - Option appOpt = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false ) - .withDescription( "Application id or app name" ).create( APPLICATION_ARG ); - - Option collOpt = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false ) - .withDescription( "Collection name" ).create( COLLECTION_ARG ); - - Option allOpt = OptionBuilder.withType( Boolean.class ) - .withArgName( ALL_ARG ).hasArg().isRequired( false ) - .withDescription( "True to reindex all application" ).create( ALL_ARG ); - - Options options = new Options(); - options.addOption( hostOpt ); - options.addOption( esHostsOpt ); - options.addOption( esClusterOpt ); - options.addOption( appOpt ); - options.addOption( collOpt ); - options.addOption( allOpt ); - - return options; - } - - - /* - * (non-Javadoc) - * - * @see - * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine) - */ - @Override - public void runTool( CommandLine line ) throws Exception { - startSpring(); - - logger.info( "Starting index rebuild" ); - - EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() { - - @Override - public void onProgress(EntityRef entity) { - logger.info("Indexing entity {}:{}", entity.getType(), entity.getUuid()); - } - - @Override - public long getWriteDelayTime() { - return 100; - } - }; - - emf.rebuildInternalIndexes( po ); - emf.refreshIndex(); - - if ( line.getOptionValue("all") != null && line.getOptionValue("all").equalsIgnoreCase("true") ) { - emf.rebuildAllIndexes( po ); - - } else if ( line.getOptionValue( APPLICATION_ARG ) != null ) { - - // Goes through each app id specified - for ( UUID appId : getAppIds( line ) ) { - - logger.info( "Reindexing for app id: {}", appId ); - Set<String> collections = getCollections( line, appId ); - - for ( String collection : collections ) { - emf.rebuildCollectionIndex( appId, collection, po ); - emf.refreshIndex(); - } - } - - } else { - - Map<String, UUID> ids = emf.getApplications(); - System.out.println( "Printing all apps" ); - for ( Entry<String, UUID> entry : ids.entrySet() ) { - System.out.println( entry.getKey() + " appid=" + entry.getValue() ); - } - } - - logger.info( "Finished index rebuild" ); - } - - - /** Get all app id */ - private Collection<UUID> getAppIds( CommandLine line ) throws Exception { - - String appId = line.getOptionValue( APPLICATION_ARG ); - - Map<String, UUID> ids = emf.getApplications(); - - if ( appId != null ) { - UUID id = UUIDUtils.tryExtractUUID( appId ); - if ( id == null ) { - logger.debug("Got applications: " + ids ); - id = emf.getApplications().get( appId ); - } - return Collections.singleton( id ); - } - - return ids.values(); - } - - - /** Get collection names. If none are specified, all are returned */ - private Set<String> getCollections( CommandLine line, UUID appId ) throws Exception { - - String passedName = line.getOptionValue( COLLECTION_ARG ); - - if ( passedName != null ) { - return Collections.singleton( passedName ); - } - - EntityManager em = emf.getEntityManager( appId ); - - return em.getApplicationCollections(); - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java index b308290..a056f77 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import org.apache.usergrid.persistence.index.query.CounterResolution; import org.codehaus.jackson.JsonGenerator; import org.apache.usergrid.management.ApplicationInfo; import org.apache.usergrid.management.OrganizationInfo; @@ -46,7 +47,6 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.BiMap; import com.google.common.collect.ListMultimap; import com.google.common.collect.Ordering; -import org.apache.usergrid.persistence.index.query.CounterResolution; /** http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java b/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java index 4fbccb2..4493b67 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java @@ -25,7 +25,7 @@ import java.util.UUID; import org.apache.usergrid.management.UserInfo; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; -import org.apache.usergrid.persistence.index.query.Query; +import org.apache.usergrid.persistence.Query; import org.apache.usergrid.persistence.Results; import org.apache.commons.cli.CommandLine; http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java b/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java index b67e9eb..48e2af6 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java @@ -46,7 +46,7 @@ import static org.apache.usergrid.services.ServicePayload.payload; public class PopulateSample extends ToolBase { - private static final Logger logger = LoggerFactory.getLogger( Export.class ); + private static final Logger logger = LoggerFactory.getLogger( PopulateSample.class ); @Override http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/RepairingMismatchedApplicationMetadata.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/RepairingMismatchedApplicationMetadata.java b/stack/tools/src/main/java/org/apache/usergrid/tools/RepairingMismatchedApplicationMetadata.java deleted file mode 100644 index 7b18ccc..0000000 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/RepairingMismatchedApplicationMetadata.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.tools; - - -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Options; -import org.apache.log4j.Level; -import org.apache.log4j.LogManager; - -import org.apache.usergrid.management.OrganizationInfo; - -import com.google.common.collect.BiMap; - -import me.prettyprint.cassandra.serializers.ByteBufferSerializer; -import me.prettyprint.hector.api.Keyspace; -import me.prettyprint.hector.api.mutation.Mutator; - -import static me.prettyprint.hector.api.factory.HFactory.createMutator; -import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME; -import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID; -import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator; -import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.batchExecute; -import static org.apache.usergrid.persistence.cassandra.CassandraService.APPLICATIONS_CF; -import static org.apache.usergrid.persistence.cassandra.CassandraService.RETRY_COUNT; - - -public class RepairingMismatchedApplicationMetadata extends ToolBase { - - public static final ByteBufferSerializer be = new ByteBufferSerializer(); - - private static final Logger logger = LoggerFactory.getLogger( RepairingMismatchedApplicationMetadata.class ); - - - @Override - public Options createOptions() { - Options options = super.createOptions(); - return options; - } - - - @Override - public void runTool( CommandLine line ) throws Exception { - startSpring(); - - //sucks, but it's not picking up the configuration - LogManager.getLogger( RepairingMismatchedApplicationMetadata.class ).setLevel( Level.INFO ); - - UUID orgId = null; - List<OrganizationInfo> orgs; - - final int size = 1000; - - - do { - orgs = managementService.getOrganizations( orgId, size ); - - - for ( OrganizationInfo org : orgs ) { - - orgId = org.getUuid(); - - logger.info( "Auditing org {}", org.getName() ); - - try { - BiMap<UUID, String> apps = managementService.getApplicationsForOrganization( org.getUuid() ); - - - for ( Map.Entry<UUID, String> app : apps.entrySet() ) { - - logger.info( "Auditing org {} app {}", org.getName(), app.getValue() ); - - UUID applicationId = emf.lookupApplication( app.getValue() ); - if ( applicationId == null ) { - String appName = app.getValue(); - Keyspace ko = cass.getSystemKeyspace(); - Mutator<ByteBuffer> m = createMutator( ko, be ); - long timestamp = cass.createTimestamp(); - addInsertToMutator( m, APPLICATIONS_CF, appName, PROPERTY_UUID, app.getKey(), timestamp ); - addInsertToMutator( m, APPLICATIONS_CF, appName, PROPERTY_NAME, appName, timestamp ); - batchExecute( m, RETRY_COUNT ); - logger.info( "Repairing alias with app uuid {}, and name {}", app.getKey(), - app.getValue() ); - } - } - } - catch ( Exception e ) { - logger.error( "Unable to process applications for organization {}", org, e ); - } - } - } - while ( orgs != null && orgs.size() == size ); - - logger.info( "Completed repairing aliases" ); - Thread.sleep( 1000 * 60 ); - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java index aa33e49..7fb7015 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java @@ -101,6 +101,7 @@ public abstract class ToolBase { System.setProperty( "cassandra.url", line.getOptionValue( "host" ) ); System.setProperty( "elasticsearch.hosts", line.getOptionValue( "eshost" ) ); System.setProperty( "elasticsearch.cluster_name", line.getOptionValue( "escluster" ) ); + System.setProperty( "usergrid.cluster_name", "usergrid" ); } try { @@ -134,6 +135,12 @@ public abstract class ToolBase { Option hostOption = OptionBuilder.withArgName( "host" ).hasArg() .withDescription( "Cassandra host" ).create( "host" ); + Option esHostOption = OptionBuilder.withArgName( "eshost" ).hasArg() + .withDescription( "ElasticSearch host" ).create( "eshost" ); + + Option esClusterOption = OptionBuilder.withArgName( "escluster" ).hasArg() + .withDescription( "ElasticSearch cluster name" ).create( "escluster" ); + Option remoteOption = OptionBuilder .withDescription( "Use remote Cassandra instance" ).create( "remote" ); @@ -143,6 +150,8 @@ public abstract class ToolBase { Options options = new Options(); options.addOption( hostOption ); + options.addOption( esHostOption ); + options.addOption( esClusterOption ); options.addOption( remoteOption ); options.addOption( verbose ); @@ -181,7 +190,7 @@ public abstract class ToolBase { Setup setup = ( (CpEntityManagerFactory) emf ).getSetup(); logger.info( "Setting up Usergrid schema" ); - setup.init(); + setup.initSubsystems(); logger.info( "Usergrid schema setup" ); logger.info( "Setting up Usergrid management services" ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java deleted file mode 100644 index 0ae45d6..0000000 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.tools; - - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.UUID; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.usergrid.management.ApplicationInfo; -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.EntityManager; -import org.apache.usergrid.persistence.EntityManagerFactory; -import org.apache.usergrid.persistence.index.query.Identifier; -import org.apache.usergrid.persistence.IndexBucketLocator; -import org.apache.usergrid.persistence.IndexBucketLocator.IndexType; -import org.apache.usergrid.persistence.cassandra.CassandraService; -import org.apache.usergrid.persistence.cassandra.EntityManagerImpl; -import org.apache.usergrid.persistence.cassandra.index.IndexScanner; -import org.apache.usergrid.persistence.entities.Application; -import org.apache.usergrid.persistence.query.ir.result.ScanColumn; -import org.apache.usergrid.persistence.query.ir.result.SliceIterator; -import org.apache.usergrid.persistence.query.ir.result.UUIDIndexSliceParser; -import org.apache.usergrid.persistence.schema.CollectionInfo; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; - -import me.prettyprint.hector.api.Keyspace; -import me.prettyprint.hector.api.beans.AbstractComposite.ComponentEquality; -import me.prettyprint.hector.api.beans.DynamicComposite; -import me.prettyprint.hector.api.beans.HColumn; -import me.prettyprint.hector.api.mutation.Mutator; - -import static me.prettyprint.hector.api.factory.HFactory.createMutator; -import static org.apache.usergrid.persistence.Schema.DICTIONARY_COLLECTIONS; -import static org.apache.usergrid.persistence.Schema.getDefaultSchema; -import org.apache.usergrid.persistence.SimpleEntityRef; -import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX; -import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX_ENTRIES; -import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator; -import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key; -import static org.apache.usergrid.persistence.cassandra.CassandraService.INDEX_ENTRY_LIST_COUNT; -import static org.apache.usergrid.utils.CompositeUtils.setEqualityFlag; -import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros; -import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID; -import static org.apache.usergrid.persistence.cassandra.Serializers.*; - - -/** - * This is a utility to audit all available entity ids in the secondary index. It then checks to see if any index value - * is not present in the Entity_Index_Entries. If it is not, the value from the index is removed, and a forced re-index - * is triggered - * <p/> - * USERGRID-323 - * <p/> - * <p/> - * UniqueIndexCleanup -app [appid] -col [collectionname] - * - * @author tnine - */ -public class UniqueIndexCleanup extends ToolBase { - - /** - * - */ - private static final int PAGE_SIZE = 100; - - - - private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class ); - - /** - * - */ - private static final String APPLICATION_ARG = "app"; - - /** - * - */ - private static final String COLLECTION_ARG = "col"; - - - @Override - @SuppressWarnings("static-access") - public Options createOptions() { - - - Options options = new Options(); - - Option hostOption = - OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" ) - .create( "host" ); - - - options.addOption( hostOption ); - - - Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false ) - .withDescription( "application id or app name" ).create( APPLICATION_ARG ); - - - options.addOption( appOption ); - - Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false ) - .withDescription( "colleciton name" ).create( COLLECTION_ARG ); - - options.addOption( collectionOption ); - - return options; - } - - - /* - * (non-Javadoc) - * - * @see - * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine) - */ - @Override - public void runTool( CommandLine line ) throws Exception { - startSpring(); - - logger.info( "Starting entity cleanup" ); - - Map<String, UUID> apps = getApplications( emf, line ); - - - for ( Entry<String, UUID> app : apps.entrySet() ) { - - logger.info( "Starting cleanup for app {}", app.getKey() ); - - UUID applicationId = app.getValue(); - EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId ); - - //sanity check for corrupt apps - Application appEntity = em.getApplication(); - - if ( appEntity == null ) { - logger.warn( "Application does not exist in data. {}", app.getKey() ); - continue; - } - - CassandraService cass = em.getCass(); - IndexBucketLocator indexBucketLocator = em.getIndexBucketLocator(); - - Keyspace ko = cass.getApplicationKeyspace( applicationId ); - - UUID timestampUuid = newTimeUUID(); - long timestamp = getTimestampInMicros( timestampUuid ); - - - // go through each collection and audit the values - for ( String collectionName : getCollectionNames( em, line ) ) { - - - IndexScanner scanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ), - key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false, - indexBucketLocator, applicationId, collectionName, false ); - - SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() ); - - - while ( itr.hasNext() ) { - - Set<ScanColumn> ids = itr.next(); - - CollectionInfo collection = getDefaultSchema().getCollection( "application", collectionName ); - - - //We shouldn't have to do this, but otherwise the cursor won't work - Set<String> indexed = collection.getPropertiesIndexed(); - - // what's left needs deleted, do so - - logger.info( "Auditing {} entities for collection {} in app {}", new Object[] { - ids.size(), collectionName, app.getValue() - } ); - - for ( ScanColumn col : ids ) { - final UUID id = col.getUUID(); - String type = getDefaultSchema().getCollectionType("application", collectionName); - - boolean reIndex = false; - - Mutator<ByteBuffer> m = createMutator( ko, be ); - - try { - - for ( String prop : indexed ) { - - String bucket = - indexBucketLocator.getBucket( applicationId, IndexType.COLLECTION, id, prop ); - - Object rowKey = key( applicationId, collection.getName(), prop, bucket ); - - List<HColumn<ByteBuffer, ByteBuffer>> indexCols = - scanIndexForAllTypes( ko, indexBucketLocator, applicationId, rowKey, id, prop ); - - // loop through the indexed values and verify them as present in - // our entity_index_entries. If they aren't, we need to delete the - // from the secondary index, and mark - // this object for re-index via n update - for ( HColumn<ByteBuffer, ByteBuffer> index : indexCols ) { - - DynamicComposite secondaryIndexValue = - DynamicComposite.fromByteBuffer( index.getName().duplicate() ); - - Object code = secondaryIndexValue.get( 0 ); - Object propValue = secondaryIndexValue.get( 1 ); - UUID timestampId = ( UUID ) secondaryIndexValue.get( 3 ); - - DynamicComposite existingEntryStart = - new DynamicComposite( prop, code, propValue, timestampId ); - DynamicComposite existingEntryFinish = - new DynamicComposite( prop, code, propValue, timestampId ); - - setEqualityFlag( existingEntryFinish, ComponentEquality.GREATER_THAN_EQUAL ); - - // now search our EntityIndexEntry for previous values, see if - // they don't match this one - - List<HColumn<ByteBuffer, ByteBuffer>> entries = - cass.getColumns( ko, ENTITY_INDEX_ENTRIES, id, existingEntryStart, - existingEntryFinish, INDEX_ENTRY_LIST_COUNT, false ); - - // we wouldn't find this column in our entity_index_entries - // audit. Delete it, then mark this entity for update - if ( entries.size() == 0 ) { - logger.info( - "Could not find reference to value '{}' property '{}'"+ - " on entity {} in collection {}. " + " Forcing reindex", - new Object[] { propValue, prop, id, collectionName } ); - - addDeleteToMutator( - m, ENTITY_INDEX, rowKey, index.getName().duplicate(), timestamp ); - - reIndex = true; - } - - if ( entries.size() > 1 ) { - logger.info( - "Found more than 1 entity referencing unique index " - + "for property '{}' with value " + "'{}'", - prop, propValue ); - reIndex = true; - } - } - } - - //force this entity to be updated - if ( reIndex ) { - Entity entity = em.get( new SimpleEntityRef( type, id )); - - //entity may not exist, but we should have deleted rows from the index - if ( entity == null ) { - logger.warn( "Entity with id {} did not exist in app {}", - id, applicationId ); - //now execute the cleanup. In this case the entity is gone, - // so we'll want to remove references from - // the secondary index - m.execute(); - continue; - } - - - logger.info( "Reindex complete for entity with id '{} ", id ); - em.update( entity ); - - //now execute the cleanup. This way if the above update fails, - // we still have enough data to run again - // later - m.execute(); - } - } - catch ( Exception e ) { - logger.error( "Unable to process entity with id '{}'", id, e ); - } - } - } - } - } - - logger.info( "Completed audit of apps" ); - } - - - private Map<String, UUID> getApplications( EntityManagerFactory emf, CommandLine line ) throws Exception { - String appName = line.getOptionValue( APPLICATION_ARG ); - - if ( appName == null ) { - return emf.getApplications(); - } - - ApplicationInfo app = managementService.getApplicationInfo( Identifier.from( appName ) ); - - if ( app == null ) { - logger.error( "Could not find application with id or name {}", appName ); - System.exit( 3 ); - } - - - Map<String, UUID> apps = new HashMap<String, UUID>(); - - apps.put( app.getName(), app.getId() ); - - return apps; - } - - - private Set<String> getCollectionNames( EntityManager em, CommandLine line ) throws Exception { - - String collectionName = line.getOptionValue( COLLECTION_ARG ); - - if ( collectionName == null ) { - return em.getApplicationCollections(); - } - - - Set<String> names = new HashSet<String>(); - names.add( collectionName ); - - return names; - } - - - private List<HColumn<ByteBuffer, ByteBuffer>> scanIndexForAllTypes( Keyspace ko, - IndexBucketLocator indexBucketLocator, - UUID applicationId, Object rowKey, - UUID entityId, String prop ) throws Exception { - - //TODO Determine the index bucket. Scan the entire index for properties with this entityId. - - - DynamicComposite start = null; - - List<HColumn<ByteBuffer, ByteBuffer>> cols; - - List<HColumn<ByteBuffer, ByteBuffer>> results = new ArrayList<HColumn<ByteBuffer, ByteBuffer>>(); - - - do { - cols = cass.getColumns( ko, ENTITY_INDEX, rowKey, start, null, 100, false ); - - for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) { - DynamicComposite secondaryIndexValue = DynamicComposite.fromByteBuffer( col.getName().duplicate() ); - - UUID storedId = ( UUID ) secondaryIndexValue.get( 2 ); - - //add it to the set. We can't short circuit due to property ordering - if ( entityId.equals( storedId ) ) { - results.add( col ); - } - - start = secondaryIndexValue; - } - } - while ( cols.size() == 100 ); - - return results; - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java index 3b5383d..e4195a4 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java @@ -18,6 +18,7 @@ package org.apache.usergrid.tools; import com.google.common.collect.BiMap; +import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.management.UserInfo; import org.apache.commons.cli.CommandLine; @@ -65,7 +66,7 @@ public class UserManager extends ToolBase { logger.info( mapToFormattedJsonString( orgs ) ); logger.info("--- User dictionaries:"); - EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID ); + EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ); User user = em.get( userInfo.getUuid(), User.class ); Set<String> dictionaries = em.getDictionaries( user ); for (String dictionary : dictionaries) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java index 7b60257..48d47be 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java @@ -17,63 +17,42 @@ package org.apache.usergrid.tools; +import au.com.bytecode.opencsv.CSVReader; +import au.com.bytecode.opencsv.CSVWriter; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.PutObjectResult; +import com.google.common.collect.BiMap; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Module; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.usergrid.management.OrganizationInfo; +import org.apache.usergrid.persistence.*; +import org.apache.usergrid.persistence.entities.Application; +import org.apache.usergrid.persistence.schema.CollectionInfo; +import org.apache.usergrid.utils.InflectionUtils; +import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule; +import org.jclouds.logging.log4j.config.Log4JLoggingModule; +import org.jclouds.netty.config.NettyPayloadModule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.FilenameFilter; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; -import java.util.TreeMap; -import java.util.UUID; import java.util.concurrent.TimeUnit; -import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.AsyncBlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; -import org.jclouds.blobstore.domain.BlobBuilder; -import org.jclouds.blobstore.options.PutOptions; -import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule; -import org.jclouds.logging.log4j.config.Log4JLoggingModule; -import org.jclouds.netty.config.NettyPayloadModule; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.usergrid.management.OrganizationInfo; -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.EntityManager; -import org.apache.usergrid.persistence.index.query.Query; -import org.apache.usergrid.persistence.Results; -import org.apache.usergrid.persistence.Schema; -import org.apache.usergrid.persistence.entities.Application; -import org.apache.usergrid.persistence.schema.CollectionInfo; -import org.apache.usergrid.utils.InflectionUtils; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; - -import com.google.common.collect.BiMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.inject.Module; - -import au.com.bytecode.opencsv.CSVReader; -import au.com.bytecode.opencsv.CSVWriter; - import static org.apache.usergrid.persistence.Schema.getDefaultSchema; -import org.apache.usergrid.persistence.index.query.Query.Level; /** @@ -84,7 +63,7 @@ import org.apache.usergrid.persistence.index.query.Query.Level; */ public class WarehouseExport extends ExportingToolBase { - private static final Logger LOG = LoggerFactory.getLogger( WarehouseExport.class ); + private static final Logger logger = LoggerFactory.getLogger( WarehouseExport.class ); private static final char SEPARATOR = '|'; public static final String BUCKET_PROPNAME = "usergrid.warehouse-export-bucket"; @@ -136,14 +115,14 @@ public class WarehouseExport extends ExportingToolBase { applyOrgId( line ); prepareBaseOutputFileName( line ); outputDir = createOutputParentDir(); - LOG.info( "Export directory: {}", outputDir.getAbsolutePath() ); + logger.info( "Export directory: {}", outputDir.getAbsolutePath() ); // create writer applyStartTime( line ); applyEndTime( line ); - LOG.error( "startTime: {}, endTime: {}", startTime, endTime ); + logger.error( "startTime: {}, endTime: {}", startTime, endTime ); if ( startTime.getTime() >= endTime.getTime() ) { - LOG.error( "startTime must be before endTime. exiting." ); + logger.error( "startTime must be before endTime. exiting." ); System.exit( 1 ); } @@ -175,7 +154,7 @@ public class WarehouseExport extends ExportingToolBase { // now that file is written, copy it to S3 if ( line.hasOption( "upload" ) ) { - LOG.info( "Copy to S3" ); + logger.info( "Copy to S3" ); copyToS3( fileName ); } } @@ -195,40 +174,16 @@ public class WarehouseExport extends ExportingToolBase { .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule() ); - BlobStoreContext context = - ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES ) - .overrides( overrides ).buildView( BlobStoreContext.class ); - - // Create Container (the bucket in s3) - try { - AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed to sync - // BlobStore (returns false if it already exists) - ListenableFuture<Boolean> container = blobStore.createContainerInLocation( null, bucketName ); - if ( container.get() ) { - LOG.info( "Created bucket " + bucketName ); - } - } - catch ( Exception ex ) { - logger.error( "Could not start binary service: {}", ex.getMessage() ); - throw new RuntimeException( ex ); - } - - try { - File file = new File( fileName ); - AsyncBlobStore blobStore = context.getAsyncBlobStore(); - BlobBuilder blobBuilder = - blobStore.blobBuilder( file.getName() ).payload( file ).calculateMD5().contentType( "text/plain" ) - .contentLength( file.length() ); - - Blob blob = blobBuilder.build(); + AWSCredentials credentials = new BasicAWSCredentials(accessId, secretKey); + ClientConfiguration clientConfig = new ClientConfiguration(); + clientConfig.setProtocol( Protocol.HTTP); - ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() ); + AmazonS3Client s3Client = new AmazonS3Client(credentials, clientConfig); - LOG.info( "Uploaded file etag=" + futureETag.get() ); - } - catch ( Exception e ) { - LOG.error( "Error uploading to blob store", e ); - } + s3Client.createBucket( bucketName ); + File uploadFile = new File( fileName ); + PutObjectResult putObjectResult = s3Client.putObject( bucketName, uploadFile.getName(), uploadFile ); + logger.info("Uploaded file etag={}", putObjectResult.getETag()); } @@ -386,7 +341,7 @@ public class WarehouseExport extends ExportingToolBase { OrganizationInfo info = managementService.getOrganizationByUuid( orgId ); if ( info == null ) { - LOG.error( "Organization info is null!" ); + logger.error( "Organization info is null!" ); System.exit( 1 ); } @@ -422,7 +377,7 @@ public class WarehouseExport extends ExportingToolBase { private void exportApplicationsForOrg( Entry<UUID, String> orgIdAndName, String queryString ) throws Exception { - LOG.info( "organization: {} / {}", orgIdAndName.getValue(), orgIdAndName.getKey() ); + logger.info( "organization: {} / {}", orgIdAndName.getValue(), orgIdAndName.getKey() ); String orgName = orgIdAndName.getValue(); @@ -432,7 +387,7 @@ public class WarehouseExport extends ExportingToolBase { String appName = appIdAndName.getValue(); appName = appName.substring( appName.indexOf( '/' ) + 1 ); - LOG.info( "application {} / {}", appName, appIdAndName.getKey() ); + logger.info( "application {} / {}", appName, appIdAndName.getKey() ); EntityManager em = emf.getEntityManager( appIdAndName.getKey() ); Map<String, String[]> cfm = getCollectionFieldMap(); @@ -453,7 +408,7 @@ public class WarehouseExport extends ExportingToolBase { Query query = Query.fromQL( queryString ); query.setLimit( MAX_ENTITY_FETCH ); - query.setResultsLevel( Level.REFS ); + query.setResultsLevel( Query.Level.REFS ); Results results = em.searchCollection( em.getApplicationRef(), collectionName, query ); while ( results.size() > 0 ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseUpsert.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseUpsert.java b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseUpsert.java index 158abd6..901fb7e 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseUpsert.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseUpsert.java @@ -31,7 +31,7 @@ import org.apache.commons.io.IOUtils; /** Upserts data from files found in an S3 bucket. */ public class WarehouseUpsert extends ExportingToolBase { - private static final Logger LOG = LoggerFactory.getLogger( WarehouseUpsert.class ); + private static final Logger logger = LoggerFactory.getLogger( WarehouseUpsert.class ); public static final String DBHOST_PROPNAME = "usergrid.warehouse-export-dbhost"; public static final String DBPORT_PROPNAME = "usergrid.warehouse-export-dbport"; @@ -84,14 +84,14 @@ public class WarehouseUpsert extends ExportingToolBase { String mainTableName = ( String ) properties.get( MAIN_TABLE_PROPNAME ); try { con.createStatement().execute( createWarehouseTable( mainTableName ) ); - LOG.info( "Created main table " + mainTableName ); + logger.info( "Created main table " + mainTableName ); } catch ( SQLException ex ) { if ( !ex.getMessage().contains( "already exists" ) ) { - LOG.error( "Error creating main table: " + ex.getMessage(), ex ); + logger.error( "Error creating main table: " + ex.getMessage(), ex ); } else { - LOG.info( "Using existing main table " + mainTableName ); + logger.info( "Using existing main table " + mainTableName ); } } @@ -100,47 +100,47 @@ public class WarehouseUpsert extends ExportingToolBase { String dropStagingTable = String.format( "drop table %s", stagingTableName ); try { con.createStatement().execute( dropStagingTable ); - LOG.info( "Dropped existing staging table " + stagingTableName ); + logger.info( "Dropped existing staging table " + stagingTableName ); } catch ( SQLException ex ) { if ( !ex.getMessage().contains( "does not exist" ) ) { - LOG.error( "Error dropping staging table: " + ex.getMessage(), ex ); + logger.error( "Error dropping staging table: " + ex.getMessage(), ex ); } else { - LOG.info( "Using existing staging table " + stagingTableName ); + logger.info( "Using existing staging table " + stagingTableName ); } } // create staging table - LOG.info( "Creating new staging table" ); + logger.info( "Creating new staging table" ); con.createStatement().execute( createWarehouseTable( stagingTableName ) ); // copy data from S3 into staging table - LOG.info( "Copying data from S3" ); + logger.info( "Copying data from S3" ); String copyFromS3 = String.format( "COPY %s FROM 's3://%s' " + "CREDENTIALS 'aws_access_key_id=%s;aws_secret_access_key=%s' IGNOREHEADER 2 EMPTYASNULL", stagingTableName, bucketName, accessId, secretKey ); - LOG.debug( copyFromS3 ); + logger.debug( copyFromS3 ); con.createStatement().execute( copyFromS3 ); // run update portion of upsert process - LOG.info( "Upsert: updating" ); + logger.info( "Upsert: updating" ); String upsertUpdate = String.format( "UPDATE %s SET id = s.id FROM %s s WHERE %s.created = s.created ", mainTableName, stagingTableName, mainTableName ); - LOG.debug( upsertUpdate ); + logger.debug( upsertUpdate ); con.createStatement().execute( upsertUpdate ); // insert new values in staging table into main table - LOG.info( "Upsert: inserting" ); + logger.info( "Upsert: inserting" ); String upsertInsert = String.format( "INSERT INTO %s SELECT s.* FROM %s s LEFT JOIN %s n ON s.id = n.id WHERE n.id IS NULL", mainTableName, stagingTableName, mainTableName ); - LOG.debug( upsertInsert ); + logger.debug( upsertInsert ); con.createStatement().execute( upsertInsert ); // drop staging table - LOG.info( "Dropping existing staging table" ); + logger.info( "Dropping existing staging table" ); con.createStatement().execute( dropStagingTable ); // done! http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java b/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java index b0f4f6e..49b110b 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java @@ -17,19 +17,14 @@ package org.apache.usergrid.tools.bean; +import com.google.common.base.Preconditions; +import org.apache.usergrid.persistence.*; +import org.apache.usergrid.persistence.index.query.CounterResolution; + import java.util.ArrayList; import java.util.List; import java.util.UUID; -import org.apache.usergrid.persistence.AggregateCounter; -import org.apache.usergrid.persistence.AggregateCounterSet; -import org.apache.usergrid.persistence.EntityManager; -import org.apache.usergrid.persistence.index.query.Query; -import org.apache.usergrid.persistence.Results; - -import com.google.common.base.Preconditions; -import org.apache.usergrid.persistence.index.query.CounterResolution; - /** @author zznate */ public class MetricQuery { http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java index 446aa91..2946ddb 100644 --- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java +++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java @@ -17,27 +17,12 @@ package org.apache.usergrid.tools; import org.apache.commons.lang.RandomStringUtils; -import org.apache.usergrid.ServiceITSetup; -import org.apache.usergrid.ServiceITSetupImpl; -import org.apache.usergrid.ServiceITSuite; -import org.apache.usergrid.management.ApplicationInfo; -import org.apache.usergrid.management.OrganizationOwnerInfo; -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.EntityManager; -import org.junit.ClassRule; +import org.apache.usergrid.services.AbstractServiceIT; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import rx.Scheduler; -import rx.schedulers.Schedulers; import java.io.File; import java.io.FileFilter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -46,47 +31,45 @@ import static org.junit.Assert.assertTrue; /** * TODO: better test, this is really just a smoke test. */ -public class ExportAppTest { +public class ExportAppTest extends AbstractServiceIT { static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class ); - + int NUM_COLLECTIONS = 10; - int NUM_ENTITIES = 50; + int NUM_ENTITIES = 50; int NUM_CONNECTIONS = 3; - @ClassRule - public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource ); @org.junit.Test public void testBasicOperation() throws Exception { - + String rand = RandomStringUtils.randomAlphanumeric( 10 ); - + // create app with some data String orgName = "org_" + rand; String appName = "app_" + rand; - + ExportDataCreator creator = new ExportDataCreator(); creator.startTool( new String[] { "-organization", orgName, "-application", appName, - "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort() + "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort() }, false); - + long start = System.currentTimeMillis(); - + String directoryName = "target/export" + rand; ExportApp exportApp = new ExportApp(); exportApp.startTool( new String[]{ "-application", orgName + "/" + appName, "-writeThreads", "100", - "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(), + "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort(), "-outputDir", directoryName }, false ); logger.info( "100 read and 100 write threads = " + (System.currentTimeMillis() - start) / 1000 + "s" ); - + File exportDir = new File(directoryName); assertTrue( getFileCount( exportDir, "entities" ) > 0 ); assertTrue( getFileCount( exportDir, "connections" ) > 0 ); @@ -97,7 +80,7 @@ public class ExportAppTest { exportApp.startTool( new String[]{ "-application", orgName + "/" + appName, "-writeThreads", "1", - "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(), + "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort(), "-outputDir", directoryName + "1" }, false ); @@ -115,4 +98,4 @@ public class ExportAppTest { } } ).length; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java index 9cce040..ceb6074 100644 --- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java +++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java @@ -20,18 +20,16 @@ import com.google.common.collect.BiMap; import com.google.common.io.Files; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.RandomStringUtils; -import org.apache.usergrid.ServiceITSetup; -import org.apache.usergrid.ServiceITSetupImpl; -import org.apache.usergrid.ServiceITSuite; +import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.management.OrganizationInfo; import org.apache.usergrid.management.OrganizationOwnerInfo; import org.apache.usergrid.management.UserInfo; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; +import org.apache.usergrid.services.AbstractServiceIT; import org.apache.usergrid.utils.UUIDUtils; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; -import org.junit.ClassRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,18 +37,17 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FilenameFilter; -import java.util.*; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; import static junit.framework.TestCase.assertNotNull; -import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID; import static org.junit.Assert.*; -public class ExportImportAdminsTest { +public class ExportImportAdminsTest extends AbstractServiceIT { static final Logger logger = LoggerFactory.getLogger( ExportImportAdminsTest.class ); - - @ClassRule - public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource ); @org.junit.Test public void testExportUserAndOrg() throws Exception { @@ -81,7 +78,7 @@ public class ExportImportAdminsTest { ExportAdmins exportAdmins = new ExportAdmins(); exportAdmins.startTool( new String[] { - "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(), + "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort(), "-outputDir", directoryName }, false ); @@ -148,7 +145,7 @@ public class ExportImportAdminsTest { public void testImportAdminUsersAndOrgs() throws Exception { // first: generate the data file with unique user and org IDs and names - + // data contains three users each with a unique org, one user has a duplicate email String rand1 = RandomStringUtils.randomAlphanumeric( 10 ); @@ -214,8 +211,8 @@ public class ExportImportAdminsTest { ImportAdmins importAdmins = new ImportAdmins(); importAdmins.startTool( new String[]{ - "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(), - "-inputDir", tempDir.getAbsolutePath() + "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort(), + "-inputDir", tempDir.getAbsolutePath() }, false ); // verify that users and orgs were created correctly @@ -229,7 +226,7 @@ public class ExportImportAdminsTest { assertNotNull( "org 2 exists", orgInfo2 ); List<UserInfo> org2_users = setup.getMgmtSvc().getAdminUsersForOrganization( org_uuid_2 ); assertEquals( "org2 has two users", 2, org2_users.size() ); - + OrganizationInfo orgInfo3 = setup.getMgmtSvc().getOrganizationByUuid( org_uuid_3 ); assertNotNull( "org 3 exists", orgInfo3 ); List<UserInfo> org3_users = setup.getMgmtSvc().getAdminUsersForOrganization( org_uuid_3 ); @@ -237,7 +234,7 @@ public class ExportImportAdminsTest { BiMap<UUID, String> user1_orgs = setup.getMgmtSvc().getOrganizationsForAdminUser( user_uuid_1 ); assertEquals( "user 1 has 2 orgs", 2, user1_orgs.size() ); - + BiMap<UUID, String> user2_orgs = setup.getMgmtSvc().getOrganizationsForAdminUser( user_uuid_2 ); assertEquals( "user 2 has two orgs gained one from duplicate", 2, user2_orgs.size() ); @@ -248,10 +245,10 @@ public class ExportImportAdminsTest { logger.info("EXCEPTION EXPECTED"); } - EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID ); + EntityManager em = setup.getEmf().getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ); Entity user3 = em.get( user_uuid_3 ); assertNull( "duplicate user does not exist", user3 ); } -} \ No newline at end of file +}
