Copy in just the tools changes from the master branch.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e38ee3e9 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e38ee3e9 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e38ee3e9 Branch: refs/heads/USERGRID-1047 Commit: e38ee3e94abca9cd04efd14f0b8fbd37f8e457d2 Parents: 208be57 Author: Dave Johnson <snoopd...@apache.org> Authored: Wed Jan 13 15:49:26 2016 -0500 Committer: Dave Johnson <snoopd...@apache.org> Committed: Wed Jan 13 15:49:26 2016 -0500 ---------------------------------------------------------------------- stack/tools/pom.xml | 4 +- .../org/apache/usergrid/tools/AppAudit.java | 2 +- .../java/org/apache/usergrid/tools/Cli.java | 35 +- .../apache/usergrid/tools/DupAdminRepair.java | 297 -------------- .../org/apache/usergrid/tools/DupOrgRepair.java | 263 ------------- .../usergrid/tools/EntityInsertBenchMark.java | 272 ------------- .../usergrid/tools/EntityReadBenchMark.java | 353 ----------------- .../java/org/apache/usergrid/tools/Export.java | 26 +- .../org/apache/usergrid/tools/ExportAdmins.java | 41 +- .../org/apache/usergrid/tools/ExportApp.java | 137 +++---- .../usergrid/tools/ExportDataCreator.java | 41 +- .../java/org/apache/usergrid/tools/Import.java | 11 +- .../org/apache/usergrid/tools/ImportAdmins.java | 105 ++--- .../org/apache/usergrid/tools/IndexRebuild.java | 184 --------- .../java/org/apache/usergrid/tools/Metrics.java | 2 +- .../usergrid/tools/OrganizationExport.java | 2 +- .../apache/usergrid/tools/PopulateSample.java | 2 +- .../RepairingMismatchedApplicationMetadata.java | 119 ------ .../org/apache/usergrid/tools/ToolBase.java | 11 +- .../usergrid/tools/UniqueIndexCleanup.java | 386 ------------------- .../org/apache/usergrid/tools/UserManager.java | 3 +- .../apache/usergrid/tools/WarehouseExport.java | 133 +++---- .../apache/usergrid/tools/WarehouseUpsert.java | 30 +- .../apache/usergrid/tools/bean/MetricQuery.java | 13 +- .../apache/usergrid/tools/ExportAppTest.java | 45 +-- .../usergrid/tools/ExportImportAdminsTest.java | 33 +- 26 files changed, 311 insertions(+), 2239 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/pom.xml ---------------------------------------------------------------------- diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml index d7bfc98..6e0dc50 100644 --- a/stack/tools/pom.xml +++ b/stack/tools/pom.xml @@ -21,7 +21,7 @@ <parent> <groupId>org.apache.usergrid</groupId> <artifactId>usergrid</artifactId> - <version>2.0.0-SNAPSHOT</version> + <version>2.1.0-SNAPSHOT</version> <relativePath>../</relativePath> </parent> @@ -29,7 +29,7 @@ <name>Usergrid Tools</name> <description>Command line tools for Usergrid system.</description> <packaging>jar</packaging> - <version>2.0.0-SNAPSHOT</version> + <version>2.1.0-SNAPSHOT</version> <reporting> <plugins> http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/AppAudit.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/AppAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/AppAudit.java index 8604bec..92e72da 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/AppAudit.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/AppAudit.java @@ -113,7 +113,7 @@ public class AppAudit extends ToolBase { r = em.searchCollection( rootAppRef, "groups", query ); for ( Entity entity : r.getEntities() ) { - collectionOrgs.remove( entity.getProperty( "path" ) ); + collectionOrgs.remove( entity.getProperty( "path" ).toString() ); } query.setCursor( r.getCursor() ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java index 9594403..511dcaa 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java @@ -17,6 +17,19 @@ package org.apache.usergrid.tools; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.usergrid.persistence.Query; +import org.apache.usergrid.services.*; +import org.apache.usergrid.utils.HttpUtils; +import org.apache.usergrid.utils.JsonUtils; +import org.apache.usergrid.utils.UUIDUtils; +import org.codehaus.jackson.JsonFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -25,26 +38,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import org.codehaus.jackson.JsonFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.usergrid.persistence.index.query.Query; -import org.apache.usergrid.services.ServiceAction; -import org.apache.usergrid.services.ServiceManager; -import org.apache.usergrid.services.ServiceParameter; -import org.apache.usergrid.services.ServicePayload; -import org.apache.usergrid.services.ServiceRequest; -import org.apache.usergrid.services.ServiceResults; -import org.apache.usergrid.utils.HttpUtils; -import org.apache.usergrid.utils.JsonUtils; -import org.apache.usergrid.utils.UUIDUtils; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.usergrid.persistence.index.exceptions.QueryParseException; - public class Cli extends ToolBase { @@ -79,7 +72,7 @@ public class Cli extends ToolBase { } - public void handleInput() throws QueryParseException { + public void handleInput() throws Exception { BufferedReader d = new BufferedReader( new InputStreamReader( System.in ) ); UUID applicationId = null; http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/DupAdminRepair.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DupAdminRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DupAdminRepair.java deleted file mode 100644 index 17968b7..0000000 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/DupAdminRepair.java +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.tools; - - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.usergrid.management.OrganizationInfo; -import org.apache.usergrid.management.UserInfo; -import org.apache.usergrid.persistence.*; -import org.apache.usergrid.persistence.entities.Application; -import org.apache.usergrid.persistence.entities.User; -import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException; -import org.apache.usergrid.utils.JsonUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileWriter; -import java.util.*; - - -/** - * This is a utility to load all entities in an application and re-save them, this forces the secondary indexing to be - * updated. - * - * @author tnine - */ -public class DupAdminRepair extends ExportingToolBase { - - /** - * - */ - private static final int PAGE_SIZE = 100; - - private static final Logger logger = LoggerFactory.getLogger( DupAdminRepair.class ); - - - @Override - @SuppressWarnings("static-access") - public Options createOptions() { - - Option hostOption = - OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" ) - .create( "host" ); - - Option outputOption = - OptionBuilder.withArgName( "output" ).hasArg().isRequired( true ).withDescription( "Cassandra host" ) - .create( "output" ); - - Options options = new Options(); - options.addOption( hostOption ); - options.addOption( outputOption ); - - return options; - } - - - /* - * (non-Javadoc) - * - * @see - * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine) - */ - @Override - public void runTool( CommandLine line ) throws Exception { - String outputDir = line.getOptionValue( "output" ); - - String emailsDir = String.format( "%s/emails", outputDir ); - String usernamesDir = String.format( "%s/usernames", outputDir ); - createDir( emailsDir ); - createDir( usernamesDir ); - - startSpring(); - - logger.info( "Starting crawl of all admins" ); - - EntityManager em = emf.getEntityManager( emf.getManagementAppId() ); - Application app = em.getApplication(); - - // search for all orgs - - Query query = new Query(); - query.setLimit( PAGE_SIZE ); - Results r = null; - - Multimap<String, UUID> emails = HashMultimap.create(); - Multimap<String, UUID> usernames = HashMultimap.create(); - do { - - r = em.searchCollection( app, "users", query ); - - for ( Entity entity : r.getEntities() ) { - emails.put( entity.getProperty( "email" ).toString().toLowerCase(), entity.getUuid() ); - usernames.put( entity.getProperty( "username" ).toString().toLowerCase(), entity.getUuid() ); - } - - query.setCursor( r.getCursor() ); - - logger.info( "Searching next page" ); - } - while ( r != null && r.size() == PAGE_SIZE ); - - // now go through and print out duplicate emails - - for ( String username : usernames.keySet() ) { - Collection<UUID> ids = usernames.get( username ); - - if ( ids.size() > 1 ) { - logger.info( "Found multiple users with the username {}", username ); - - // force the username to be reset to the user's email - resolveUsernameConflicts( usernamesDir, username, ids ); - } - } - - for ( String email : emails.keySet() ) { - Collection<UUID> ids = emails.get( email ); - - if ( ids.size() > 1 ) { - // get the admin the same way as the rest tier, this way the OTHER - // admins will be removed - UserInfo targetUser = managementService.getAdminUserByEmail( email ); - - if ( targetUser == null ) { - - List<UUID> tempIds = new ArrayList<UUID>( ids ); - Collections.sort( tempIds ); - - UUID toLoad = tempIds.get( 0 ); - - logger.warn( "Could not load target user by email {}, loading by UUID {} instead", email, toLoad ); - targetUser = managementService.getAdminUserByUuid( toLoad ); - - ids.remove( toLoad ); - } - - UUID targetId = targetUser.getUuid(); - - ids.remove( targetId ); - - logger.warn( "Found multiple admins with the email {}. Retaining uuid {}", email, targetId ); - - FileWriter file = new FileWriter( String.format( "%s/%s.all", emailsDir, email ) ); - - Map<String, Object> userOrganizationData = managementService.getAdminUserOrganizationData( targetId ); - - file.write( JsonUtils.mapToFormattedJsonString( userOrganizationData ) ); - - for ( UUID id : ids ) { - - userOrganizationData = managementService.getAdminUserOrganizationData( id ); - - file.write( JsonUtils.mapToFormattedJsonString( userOrganizationData ) ); - - file.write( "\n\n" ); - - mergeAdmins( emailsDir, id, targetId ); - } - - file.flush(); - file.close(); - - // force the index update after all other admins have been merged - logger.info( "Forcing re-index of admin with email {} and id {}", email, targetId ); - User targetUserEntity = em.get( targetUser.getUuid(), User.class ); - em.update( targetUserEntity ); - - FileWriter merged = new FileWriter( String.format( "%s/%s.merged", emailsDir, email ) ); - - userOrganizationData = managementService.getAdminUserOrganizationData( targetUser.getUuid() ); - - merged.write( JsonUtils.mapToFormattedJsonString( userOrganizationData ) ); - merged.flush(); - merged.close(); - } - } - - logger.info( "Repair complete" ); - } - - - /** - * When our usernames are equal, we need to check if our emails are equal. If they're not, we need to change the one - * that DOES NOT get returned on a lookup by username - */ - private void resolveUsernameConflicts( String targetDir, String userName, Collection<UUID> ids ) throws Exception { - // lookup the admin id - UserInfo existing = managementService.getAdminUserByUsername( userName ); - - if ( existing == null ) { - logger.warn( "Could not determine an admin for colliding username '{}'. Skipping", userName ); - return; - } - - ids.remove( existing.getUuid() ); - - boolean collision = false; - - EntityManager em = emf.getEntityManager( emf.getManagementAppId() ); - - for ( UUID id : ids ) { - UserInfo other = managementService.getAdminUserByUuid( id ); - - // same username and email, these will be merged later in the process, - // skip it - if ( other != null && other.getEmail() != null && other.getEmail().equals( existing.getEmail() ) ) { - logger.info( - "Users with the same username '{}' have the same email '{}'. This will be resolved later in " - + "the process, skipping", userName, existing.getEmail() ); - continue; - } - - // if we get here, the emails do not match, but the usernames do. Force - // both usernames to emails - collision = true; - - setUserName( em, other, other.getEmail() ); - } - - if ( collision ) { - setUserName( em, existing, existing.getEmail() ); - } - } - - - /** Set the username to the one provided, if we can't due to duplicate property issues, we fall back to user+uuid */ - private void setUserName( EntityManager em, UserInfo other, String newUserName ) throws Exception { - logger.info( "Setting username to {} for user with username {} and id {}", new Object[] { - newUserName, other.getUsername(), other.getUuid() - } ); - - try { - em.setProperty( new SimpleEntityRef( "user", other.getUuid() ), "username", newUserName, true ); - } - catch ( DuplicateUniquePropertyExistsException e ) { - logger.warn( "More than 1 user has the username of {}. Setting the username to their username+UUID as a " - + "fallback", newUserName ); - - setUserName( em, other, String.format( "%s-%s", other.getUsername(), other.getUuid() ) ); - } - } - - - /** Merge the source admin to the target admin by copying oranizations. Then deletes the source admin */ - private void mergeAdmins( String targetDir, UUID sourceId, UUID targetId ) throws Exception { - - EntityManager em = emf.getEntityManager( emf.getManagementAppId() ); - - User sourceUser = em.get( sourceId, User.class ); - - // may have already been deleted, do nothing - if ( sourceUser == null ) { - logger.warn( "Source admin with uuid {} does not exist in cassandra", sourceId ); - return; - } - - UserInfo targetUserInfo = managementService.getAdminUserByUuid( targetId ); - - @SuppressWarnings("unchecked") Map<String, Map<String, UUID>> sourceOrgs = - ( Map<String, Map<String, UUID>> ) managementService.getAdminUserOrganizationData( sourceId ) - .get( "organizations" ); - - for ( String orgName : sourceOrgs.keySet() ) { - UUID orgId = sourceOrgs.get( orgName ).get( "uuid" ); - - OrganizationInfo org = managementService.getOrganizationByUuid( orgId ); - - logger.info( "Adding organization {} to admin with email {} and id {}", - new Object[] { org.getName(), sourceUser.getEmail(), sourceUser.getUuid() } ); - - // copy it over to the target admin - managementService.addAdminUserToOrganization( targetUserInfo, org, false ); - } - - logger.info( "Deleting admin with email {} and id {}", sourceUser.getEmail(), sourceUser.getUuid() ); - - em.delete( sourceUser ); - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java deleted file mode 100644 index a20dda1..0000000 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.tools; - - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.usergrid.management.OrganizationInfo; -import org.apache.usergrid.management.UserInfo; -import org.apache.usergrid.persistence.*; -import org.apache.usergrid.persistence.entities.Application; -import org.apache.usergrid.utils.JsonUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileWriter; -import java.util.Collection; -import java.util.Map; -import java.util.Map.Entry; -import java.util.UUID; - - -/** - * This is a utility to load all entities in an application and re-save them, this forces the secondary indexing to be - * updated. - * - * @author tnine - */ -public class DupOrgRepair extends ExportingToolBase { - - /** - * - */ - private static final int PAGE_SIZE = 100; - - private static final Logger logger = LoggerFactory.getLogger( DupOrgRepair.class ); - - - @Override - @SuppressWarnings("static-access") - public Options createOptions() { - - Option hostOption = - OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" ) - .create( "host" ); - - Option outputOption = - OptionBuilder.withArgName( "output" ).hasArg().isRequired( true ).withDescription( "Cassandra host" ) - .create( "output" ); - - Options options = new Options(); - options.addOption( hostOption ); - options.addOption( outputOption ); - - return options; - } - - - /* - * (non-Javadoc) - * - * @see - * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine) - */ - @Override - public void runTool( CommandLine line ) throws Exception { - String outputDir = line.getOptionValue( "output" ); - - createDir( outputDir ); - - startSpring(); - - logger.info( "Starting crawl of all admins" ); - - EntityManager em = emf.getEntityManager( emf.getManagementAppId() ); - Application app = em.getApplication(); - - // search for all orgs - - Query query = new Query(); - query.setLimit( PAGE_SIZE ); - Results r = null; - - Multimap<String, UUID> orgs = HashMultimap.create(); - - do { - - r = em.searchCollection( app, "groups", query ); - - for ( Entity entity : r.getEntities() ) { - String name = entity.getProperty( "path" ).toString().toLowerCase(); - orgs.put( name, entity.getUuid() ); - } - - query.setCursor( r.getCursor() ); - - logger.info( "Searching next page" ); - } - while ( r != null && r.size() == PAGE_SIZE ); - - // now go through and print out duplicate emails - - for ( String name : orgs.keySet() ) { - Collection<UUID> ids = orgs.get( name ); - - if ( ids.size() > 1 ) { - logger.warn( "Found multiple orgs with the name {}", name ); - - // look this up the same way the REST tier does. This way we will always - // map the same way and the user will not notice a background merge - OrganizationInfo orgInfo = managementService.getOrganizationByName( name ); - - UUID targetOrgId = orgInfo.getUuid(); - - ids.remove( targetOrgId ); - - for ( UUID sourceId : ids ) { - mergeOrganizations( outputDir, sourceId, targetOrgId ); - } - } - } - - logger.info( "Merge complete" ); - } - - - /** - * Merge the source orgId into the targetId in the following way. - * <p/> - * 1) link all admins from the source org to the target org 2) link all apps from the source org to the target or 3) - * delete the target org - */ - @SuppressWarnings("unchecked") - private void mergeOrganizations( String outputDir, UUID sourceOrgId, UUID targetOrgId ) throws Exception { - - OrganizationInfo sourceOrgInfo = managementService.getOrganizationByUuid( sourceOrgId ); - - Map<String, Object> sourceOrg = managementService.getOrganizationData( sourceOrgInfo ); - - OrganizationInfo targetOrgInfo = managementService.getOrganizationByUuid( targetOrgId ); - - Map<String, Object> targetOrg = managementService.getOrganizationData( targetOrgInfo ); - - // Dump the output on these two orgs - FileWriter file = - new FileWriter( String.format( "%s/%s.%s.orig", outputDir, sourceOrgInfo.getName(), sourceOrgId ) ); - - file.write( JsonUtils.mapToFormattedJsonString( sourceOrg ) ); - - file.write( "\n\n" ); - - file.write( JsonUtils.mapToFormattedJsonString( targetOrg ) ); - - file.flush(); - file.close(); - - // BiMap<UUID, String> targetApps = - // managementService.getApplicationsForOrganization(targetOrgId); - - // now perform the merge - - // add all the admins - Map<String, UserInfo> admins = ( Map<String, UserInfo> ) sourceOrg.get( "users" ); - - for ( Entry<String, UserInfo> adminEntry : admins.entrySet() ) { - UserInfo admin = adminEntry.getValue(); - - logger.info( "adding admin with uuid {} and email {} to org with name {} and uuid {}", new Object[] { - admin.getUuid(), admin.getEmail(), targetOrgInfo.getName(), targetOrgInfo.getUuid() - } ); - - // copy the admins over - managementService.addAdminUserToOrganization( admin, targetOrgInfo, false ); - } - - // get the root entity manager - EntityManager em = emf.getEntityManager( emf.getManagementAppId() ); - - // Add the apps to the org - Map<String, UUID> sourceApps = ( Map<String, UUID> ) sourceOrg.get( "applications" ); - - Map<String, UUID> targetApps = ( Map<String, UUID> ) targetOrg.get( "applications" ); - - for ( Entry<String, UUID> app : sourceApps.entrySet() ) { - - Entity appEntity = null; - - // we have colliding app names - if ( targetApps.get( app.getKey() ) != null ) { - - // already added, skip it - if ( app.getValue().equals( targetApps.get( app.getKey() ) ) ) { - continue; - } - - // check to see if this orgname/appname lookup returns the app we're - // about to re-assign. If it does NOT, then we need to rename this app - // before performing the link. - UUID appIdToKeep = emf.lookupApplication( app.getKey() ).get(); - - UUID appIdToChange = - appIdToKeep.equals( app.getValue() ) ? targetApps.get( app.getKey() ) : app.getValue(); - - // get the existing target entity - appEntity = em.get( new SimpleEntityRef("application", appIdToChange)); - - if ( appEntity != null ) { - - String oldName = appEntity.getProperty( "name" ).toString(); - String newName = oldName + appEntity.getUuid(); - - //force the property to be updated - em.setProperty( appEntity, "name", newName, true ); - - logger.info( "Renamed app from {} to {}", oldName, newName ); - } - } - - logger.info( "Adding application with name {} and id {} to organization with uuid {}", new Object[] { - app.getKey(), app.getValue(), targetOrgId - } ); - - managementService.addApplicationToOrganization( targetOrgId, app.getValue(), appEntity); - } - - // now delete the original org - - logger.info( "Deleting org with the name {} and uuid {}", sourceOrgInfo.getName(), sourceOrgInfo.getUuid() ); - - // delete the source org - em.delete( new SimpleEntityRef( "group", sourceOrgId ) ); - - // re-dump the target from the cassandra stat - targetOrgInfo = managementService.getOrganizationByUuid( targetOrgId ); - - targetOrg = managementService.getOrganizationData( targetOrgInfo ); - - file = new FileWriter( String.format( "%s/%s.%s.new", outputDir, targetOrgInfo.getName(), targetOrgId ) ); - - file.write( JsonUtils.mapToFormattedJsonString( targetOrg ) ); - - file.flush(); - file.close(); - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java deleted file mode 100644 index fe1edbf..0000000 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.tools; - - -import java.nio.ByteBuffer; -import java.util.Stack; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.usergrid.persistence.DynamicEntity; -import org.apache.usergrid.persistence.IndexBucketLocator; -import org.apache.usergrid.persistence.IndexBucketLocator.IndexType; -import org.apache.usergrid.persistence.cassandra.EntityManagerImpl; -import org.apache.usergrid.utils.UUIDUtils; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; - -import me.prettyprint.hector.api.Keyspace; -import me.prettyprint.hector.api.beans.DynamicComposite; -import me.prettyprint.hector.api.mutation.Mutator; - -import static me.prettyprint.hector.api.factory.HFactory.createMutator; -import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX; -import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE; -import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator; -import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key; -import static org.apache.usergrid.persistence.cassandra.IndexUpdate.indexValueCode; -import static org.apache.usergrid.persistence.cassandra.Serializers.*; - - -/** - * A utility to insert entities into the em for benchmarking - * - * @author tnine - */ -public class EntityInsertBenchMark extends ToolBase { - - - private static final Logger logger = LoggerFactory.getLogger( EntityInsertBenchMark.class ); - - - @Override - @SuppressWarnings("static-access") - public Options createOptions() { - - Option hostOption = - OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" ) - .create( "host" ); - - Option countOption = - OptionBuilder.withArgName( "count" ).hasArg().isRequired( true ).withDescription( "Number of records" ) - .create( "count" ); - - Option appIdOption = OptionBuilder.withArgName( "appId" ).hasArg().isRequired( true ) - .withDescription( "Application Id to use" ).create( "appId" ); - - Option workerOption = OptionBuilder.withArgName( "workers" ).hasArg().isRequired( true ) - .withDescription( "Number of workers to use" ).create( "workers" ); - - - Options options = new Options(); - options.addOption( hostOption ); - options.addOption( countOption ); - options.addOption( appIdOption ); - options.addOption( workerOption ); - - return options; - } - - - /* - * (non-Javadoc) - * - * @see - * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine) - */ - @Override - public void runTool( CommandLine line ) throws Exception { - startSpring(); - - logger.info( "Starting entity cleanup" ); - - int workerSize = Integer.parseInt( line.getOptionValue( "workers" ) ); - - - ExecutorService executors = Executors.newFixedThreadPool( workerSize ); - - int count = Integer.parseInt( line.getOptionValue( "count" ) ); - - int size = count / workerSize; - - UUID appId = UUID.fromString( line.getOptionValue( "appId" ) ); - - Stack<Future<Void>> futures = new Stack<Future<Void>>(); - - for ( int i = 0; i < workerSize; i++ ) { - futures.push( executors.submit( new InsertWorker( i, size, appId ) ) ); - } - - System.out.println( "Waiting for workers to complete insertion" ); - - /** - * Wait for all tasks to complete - */ - while ( !futures.isEmpty() ) { - futures.pop().get(); - } - - System.out.println( "All workers completed insertion" ); - } - - - private class InsertWorker implements Callable<Void> { - - private int count; - - private int workerNumber; - - private UUID appId; - - - private InsertWorker( int workerNumber, int count, UUID appId ) { - this.workerNumber = workerNumber; - this.count = count; - this.appId = appId; - } - - - /* - * (non-Javadoc) - * - * @see java.util.concurrent.Callable#call() - */ - @Override - public Void call() throws Exception { - - Keyspace ko = EntityInsertBenchMark.this.cass.getApplicationKeyspace( appId ); - EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( appId ); - IndexBucketLocator indexBucketLocator = em.getIndexBucketLocator(); - - for ( int i = 0; i < count; i++ ) { - - Mutator<ByteBuffer> m = createMutator( ko, be ); - - DynamicEntity dynEntity = new DynamicEntity(); - dynEntity.setType( "test" ); - dynEntity.setUuid( UUIDUtils.newTimeUUID() ); - - String value = new StringBuilder().append( workerNumber ).append( "-" ).append( i ).toString(); - - - String bucketId = - indexBucketLocator.getBucket( appId, IndexType.COLLECTION, dynEntity.getUuid(), "test" ); - - Object index_name = key( appId, "tests", "test", bucketId ); - - IndexEntry entry = new IndexEntry( dynEntity.getUuid(), "test", value, UUIDUtils.newTimeUUID() ); - - addInsertToMutator( m, ENTITY_INDEX, index_name, entry.getIndexComposite(), null, - System.currentTimeMillis() ); - - UniqueIndexer indexer = new UniqueIndexer( m ); - indexer.writeIndex( appId, "tests", dynEntity.getUuid(), "test", value ); - // write this to the direct collection index - - m.execute(); - - if ( i % 100 == 0 ) { - System.out.println( - String.format( "%s : Written %d of %d", Thread.currentThread().getName(), i, count ) ); - } - } - - return null; - } - } - - - private class UniqueIndexer { - - private Mutator<ByteBuffer> mutator; - - - /** - * @param mutator - */ - public UniqueIndexer( Mutator<ByteBuffer> mutator ) { - super(); - this.mutator = mutator; - } - - - private void writeIndex( UUID applicationId, String collectionName, UUID entityId, String propName, - Object entityValue ) { - - Object rowKey = key( applicationId, collectionName, propName, entityValue ); - - addInsertToMutator( mutator, ENTITY_UNIQUE, rowKey, entityId, null, System.currentTimeMillis() ); - } - } - - - public static class IndexEntry { - private final byte code; - private String path; - private final Object value; - private final UUID timestampUuid; - private final UUID entityId; - - - public IndexEntry( UUID entityId, String path, Object value, UUID timestampUuid ) { - this.entityId = entityId; - this.path = path; - this.value = value; - code = indexValueCode( value ); - this.timestampUuid = timestampUuid; - } - - - public String getPath() { - return path; - } - - - public void setPath( String path ) { - this.path = path; - } - - - public Object getValue() { - return value; - } - - - public byte getValueCode() { - return code; - } - - - public UUID getTimestampUuid() { - return timestampUuid; - } - - - public DynamicComposite getIndexComposite() { - return new DynamicComposite( code, value, entityId, timestampUuid ); - } - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java deleted file mode 100644 index 1b54495..0000000 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java +++ /dev/null @@ -1,353 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.tools; - - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Stack; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.Assert; -import org.apache.usergrid.persistence.IndexBucketLocator; -import org.apache.usergrid.persistence.IndexBucketLocator.IndexType; -import org.apache.usergrid.persistence.cassandra.EntityManagerImpl; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; - -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.MetricPredicate; -import com.yammer.metrics.core.Timer; -import com.yammer.metrics.core.TimerContext; -import com.yammer.metrics.reporting.ConsoleReporter; - -import me.prettyprint.hector.api.Keyspace; -import me.prettyprint.hector.api.beans.AbstractComposite.ComponentEquality; -import me.prettyprint.hector.api.beans.DynamicComposite; -import me.prettyprint.hector.api.beans.HColumn; -import me.prettyprint.hector.api.beans.Row; -import me.prettyprint.hector.api.beans.Rows; -import me.prettyprint.hector.api.factory.HFactory; -import me.prettyprint.hector.api.query.MultigetSliceQuery; -import me.prettyprint.hector.api.query.QueryResult; - -import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX; -import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE; -import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key; -import static org.apache.usergrid.persistence.cassandra.IndexUpdate.indexValueCode; -import static org.apache.usergrid.utils.ConversionUtils.bytebuffers; -import static org.apache.usergrid.persistence.cassandra.Serializers.*; - - -/** - * A utility to insert entities into the em for benchmarking - * - * @author tnine - */ -public class EntityReadBenchMark extends ToolBase { - - - - private static final Logger logger = LoggerFactory.getLogger( EntityReadBenchMark.class ); - - private final Timer queryReads = - Metrics.newTimer( ReadWorker.class, "entity", TimeUnit.MILLISECONDS, TimeUnit.SECONDS ); - - private final Timer dictReads = - Metrics.newTimer( ReadWorker.class, "dictionary", TimeUnit.MILLISECONDS, TimeUnit.SECONDS ); - - private static final String TYPE_DICTIONARY = "dict"; - private static final String TYPE_ENTITY = "entity"; - - - @Override - @SuppressWarnings("static-access") - public Options createOptions() { - - Option hostOption = - OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" ) - .create( "host" ); - - Option countOption = - OptionBuilder.withArgName( "count" ).hasArg().isRequired( true ).withDescription( "Number of records" ) - .create( "count" ); - - Option appIdOption = OptionBuilder.withArgName( "appId" ).hasArg().isRequired( true ) - .withDescription( "Application Id to use" ).create( "appId" ); - - Option workerOption = OptionBuilder.withArgName( "workers" ).hasArg().isRequired( true ) - .withDescription( "Number of workers to use" ).create( "workers" ); - - - Option typeOption = OptionBuilder.withArgName( "type" ).hasArg().isRequired( true ) - .withDescription( "Read type to use, 'dict' or 'entity'" ).create( "type" ); - - Options options = new Options(); - options.addOption( hostOption ); - options.addOption( countOption ); - options.addOption( appIdOption ); - options.addOption( workerOption ); - options.addOption( typeOption ); - - return options; - } - - - /* - * (non-Javadoc) - * - * @see - * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine) - */ - @Override - public void runTool( CommandLine line ) throws Exception { - startSpring(); - - logger.info( "Starting entity cleanup" ); - - int workerSize = Integer.parseInt( line.getOptionValue( "workers" ) ); - - ExecutorService executors = Executors.newFixedThreadPool( workerSize ); - - int count = Integer.parseInt( line.getOptionValue( "count" ) ); - - int size = count / workerSize; - - UUID appId = UUID.fromString( line.getOptionValue( "appId" ) ); - - System.out.println( "Querying unique properties in the search index" ); - - - final ConsoleReporter reporter = - new ConsoleReporter( Metrics.defaultRegistry(), System.out, MetricPredicate.ALL ); - - //print every 30 seconds - reporter.start( 30, TimeUnit.SECONDS ); - - Stack<Future<Void>> futures = new Stack<Future<Void>>(); - - - String type = line.getOptionValue( "type" ); - - for ( int i = 0; i < workerSize; i++ ) { - - ReadWorker worker = null; - - - if ( TYPE_ENTITY.equals( type ) ) { - worker = new IndexReadWorker( i, size, appId ); - } - else if ( TYPE_DICTIONARY.equals( type ) ) { - worker = new DictReadWorker( i, size, appId ); - } - else { - throw new IllegalArgumentException( "You must specifiy the 'type' option" ); - } - - futures.push( executors.submit( worker ) ); - } - - - System.out.println( "Waiting for index read workers to complete" ); - - /** - * Wait for all tasks to complete - */ - while ( !futures.isEmpty() ) { - futures.pop().get(); - } - - - System.out.println( "All workers completed reading" ); - - - //print the report - reporter.run(); - } - - - private abstract class ReadWorker implements Callable<Void> { - - protected int count; - - protected int workerNumber; - - protected UUID appId; - - - private ReadWorker( int workerNumber, int count, UUID appId ) throws Exception { - this.workerNumber = workerNumber; - this.count = count; - this.appId = appId; - } - - - /* - * (non-Javadoc) - * - * @see java.util.concurrent.Callable#call() - */ - @Override - public Void call() throws Exception { - - - for ( int i = 0; i < count; i++ ) { - - String value = new StringBuilder().append( workerNumber ).append( "-" ).append( i ).toString(); - - - doRead( value ); - } - - return null; - } - - - protected abstract void doRead( String value ) throws Exception; - } - - - private class IndexReadWorker extends ReadWorker { - - private Keyspace keyspace; - private IndexBucketLocator indexBucketLocator = null; - - - private IndexReadWorker( int workerNumber, int count, UUID appId ) throws Exception { - super( workerNumber, count, appId ); - keyspace = EntityReadBenchMark.this.cass.getApplicationKeyspace( appId ); - indexBucketLocator = ( ( EntityManagerImpl ) EntityReadBenchMark.this.emf.getEntityManager( appId ) ) - .getIndexBucketLocator(); - } - - - /* (non-Javadoc) - * @see org.apache.usergrid.tools.EntityReadBenchMark.ReadWorker#doRead() - */ - @Override - protected void doRead( String value ) throws Exception { - TimerContext timer = queryReads.time(); - - Assert.isTrue( read( value ) ); - - timer.stop(); - } - - - private boolean read( String value ) { - - - List<String> buckets = indexBucketLocator.getBuckets( appId, IndexType.UNIQUE, "tests" ); - - List<Object> cassKeys = new ArrayList<Object>( buckets.size() ); - - Object keyPrefix = key( appId, "tests", "test" ); - - for ( String bucket : buckets ) { - cassKeys.add( key( keyPrefix, bucket ) ); - } - - MultigetSliceQuery<ByteBuffer, DynamicComposite, ByteBuffer> multiget = - HFactory.createMultigetSliceQuery( keyspace, be, dce, - be ); - - multiget.setColumnFamily( ENTITY_INDEX.getColumnFamily() ); - multiget.setKeys( bytebuffers( cassKeys ) ); - - - DynamicComposite start = new DynamicComposite( indexValueCode( value ), value ); - - DynamicComposite finish = new DynamicComposite( indexValueCode( value ) ); - finish.addComponent( 1, value, ComponentEquality.GREATER_THAN_EQUAL ); - - - multiget.setRange( start, finish, false, 1 ); - QueryResult<Rows<ByteBuffer, DynamicComposite, ByteBuffer>> results = multiget.execute(); - - // search for a column, if one exists, we've found the entity - for ( Row<ByteBuffer, DynamicComposite, ByteBuffer> row : results.get() ) { - if ( row.getColumnSlice().getColumns().size() > 0 ) { - return true; - } - } - - return false; - } - } - - - private class DictReadWorker extends ReadWorker { - - - UniqueIndexer indexer; - - - private DictReadWorker( int workerNumber, int count, UUID appId ) throws Exception { - super( workerNumber, count, appId ); - Keyspace ko = EntityReadBenchMark.this.cass.getApplicationKeyspace( appId ); - indexer = new UniqueIndexer( ko ); - } - - - /* (non-Javadoc) - * @see org.apache.usergrid.tools.EntityReadBenchMark.ReadWorker#doRead() - */ - @Override - protected void doRead( String value ) throws Exception { - - TimerContext timer = dictReads.time(); - - Assert.isTrue( indexer.existsInIndex( appId, "tests", "test", value ) ); - - timer.stop(); - } - } - - - private class UniqueIndexer { - - private Keyspace keyspace; - public UniqueIndexer( Keyspace keyspace ) { - super(); - this.keyspace = keyspace; - } - - - private boolean existsInIndex( UUID applicationId, String collectionName, String propName, Object entityValue ) - throws Exception { - Object rowKey = key( applicationId, collectionName, propName, entityValue ); - - - List<HColumn<ByteBuffer, ByteBuffer>> cols = - cass.getColumns( keyspace, ENTITY_UNIQUE, rowKey, null, null, 2, false ); - - - return cols.size() > 0; - } - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java index eb616a8..bc83921 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java @@ -26,6 +26,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.UUID; +import org.apache.usergrid.persistence.*; import org.codehaus.jackson.JsonEncoding; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; @@ -33,19 +34,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.usergrid.management.OrganizationInfo; import org.apache.usergrid.management.UserInfo; -import org.apache.usergrid.persistence.ConnectionRef; -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.EntityManager; -import org.apache.usergrid.persistence.index.query.Query; -import org.apache.usergrid.persistence.Results; import org.apache.usergrid.tools.bean.ExportOrg; import org.apache.usergrid.utils.JsonUtils; import org.apache.commons.cli.CommandLine; import com.google.common.collect.BiMap; -import org.apache.usergrid.persistence.SimpleEntityRef; -import org.apache.usergrid.persistence.index.query.Query.Level; public class Export extends ExportingToolBase { @@ -175,7 +169,7 @@ public class Export extends ExportingToolBase { Query query = new Query(); query.setLimit( MAX_ENTITY_FETCH ); - query.setResultsLevel( Level.ALL_PROPERTIES ); + query.setResultsLevel( Query.Level.ALL_PROPERTIES ); Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query ); @@ -239,7 +233,7 @@ public class Export extends ExportingToolBase { // Start collection array. jg.writeStartArray(); - Results collectionMembers = em.getCollection( entity, collectionName, null, 100000, Level.IDS, false ); + Results collectionMembers = em.getCollection( entity, collectionName, null, 100000, Query.Level.IDS, false ); List<UUID> entityIds = collectionMembers.getIds(); @@ -307,13 +301,13 @@ public class Export extends ExportingToolBase { jg.writeFieldName( connectionType ); jg.writeStartArray(); - Results results = em.getConnectedEntities( - entity, connectionType, null, Level.IDS ); + Results results = em.getTargetEntities( + entity, connectionType, null, Query.Level.IDS ); List<ConnectionRef> connections = results.getConnections(); for ( ConnectionRef connectionRef : connections ) { - jg.writeObject( connectionRef.getConnectedEntity().getUuid() ); + jg.writeObject( connectionRef.getTargetRefs().getUuid() ); } jg.writeEndArray(); @@ -330,8 +324,8 @@ public class Export extends ExportingToolBase { * write entity_id : { "collectionName" : [ids] * } * } - * - * + * + * * { * entity_id : * { collection_name : @@ -348,9 +342,9 @@ public class Export extends ExportingToolBase { * ] * } * } - * + * * http://jackson.codehaus.org/1.8.0/javadoc/org/codehaus/jackson/JsonGenerator.html - * + * * *- * List<ConnectedEntityRef> connections = em.getConnections(entityId, query); http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java index 0bb74ab..479d740 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java @@ -23,8 +23,10 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.usergrid.corepersistence.util.CpNamingUtils; +import org.apache.usergrid.management.UserInfo; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; +import org.apache.usergrid.persistence.Query; import org.apache.usergrid.persistence.Results; import org.apache.usergrid.utils.StringUtils; import org.codehaus.jackson.JsonGenerator; @@ -144,12 +146,10 @@ public class ExportAdmins extends ExportingToolBase { // start read queue workers BlockingQueue<UUID> readQueue = new LinkedBlockingQueue<UUID>(); - List<AdminUserReader> readers = new ArrayList<AdminUserReader>(); for (int i = 0; i < readThreadCount; i++) { AdminUserReader worker = new AdminUserReader( readQueue, writeQueue ); Thread readerThread = new Thread( worker, "AdminUserReader-" + i ); readerThread.start(); - readers.add( worker ); } logger.debug( readThreadCount + " read worker threads started" ); @@ -205,7 +205,7 @@ public class ExportAdmins extends ExportingToolBase { logger.info( "Building org map" ); - ExecutorService execService = Executors.newFixedThreadPool( this.readThreadCount ); + ExecutorService execService = Executors.newFixedThreadPool( readThreadCount ); EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ); String queryString = "select *"; @@ -345,15 +345,15 @@ public class ExportAdmins extends ExportingToolBase { 0 : task.dictionariesByName.get( "credentials" )); logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}", - new Object[]{ - actionTaken, - task.adminUser.getProperty( "username" ), - task.adminUser.getProperty( "email" ), - task.adminUser.getUuid(), - task.orgNamesByUuid.size(), - task.dictionariesByName.size(), - creds == null ? 0 : creds.size() - } ); + new Object[]{ + actionTaken, + task.adminUser.getProperty( "username" ), + task.adminUser.getProperty( "email" ), + task.adminUser.getUuid(), + task.orgNamesByUuid.size(), + task.dictionariesByName.size(), + creds == null ? 0 : creds.size() + } ); } catch ( Exception e ) { logger.error("Error reading data for user " + uuid, e ); @@ -385,13 +385,8 @@ public class ExportAdmins extends ExportingToolBase { task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() ); -<<<<<<< HEAD - List<Org> orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() ); + List<Org> orgs = userToOrgsMap.get( task.adminUser.getUuid() ); -======= - List<Org> orgs = userToOrgsMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() ); - ->>>>>>> master if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) { // list of orgs from getOrganizationsForAdminUser() is less than expected, use userToOrgsMap @@ -401,16 +396,6 @@ public class ExportAdmins extends ExportingToolBase { } task.orgNamesByUuid = bimap; } -<<<<<<< HEAD - - if ( task.orgNamesByUuid.isEmpty() ) { - logger.error("{}:{}:{} has no orgs", new Object[] { - task.adminUser.getProperty("username"), - task.adminUser.getProperty("email"), - task.adminUser.getUuid() } ); - } -======= ->>>>>>> master } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java index db975e6..a90a83b 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java @@ -35,13 +35,15 @@ import rx.Scheduler; import rx.Subscriber; import rx.functions.Action0; import rx.functions.Action1; -import rx.functions.Func1; import rx.schedulers.Schedulers; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.*; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; @@ -49,13 +51,13 @@ import java.util.concurrent.atomic.AtomicInteger; /** * Export all entities and connections of a Usergrid app. - * + * * Exports data files to specified directory. - * + * * Will create as many output files as there are writeThreads (by default: 10). - * + * * Will create two types of files: *.entities for Usegrird entities and *.collections for entity to entity connections. - * + * * Every line of the data files is a complete JSON object. */ public class ExportApp extends ExportingToolBase { @@ -63,7 +65,7 @@ public class ExportApp extends ExportingToolBase { static final String APPLICATION_NAME = "application"; private static final String WRITE_THREAD_COUNT = "writeThreads"; - + String applicationName; String organizationName; @@ -76,7 +78,7 @@ public class ExportApp extends ExportingToolBase { Map<Thread, JsonGenerator> entityGeneratorsByThread = new HashMap<Thread, JsonGenerator>(); Map<Thread, JsonGenerator> connectionGeneratorsByThread = new HashMap<Thread, JsonGenerator>(); - int writeThreadCount = 10; // set via CLI option; limiting write will limit output files + int writeThreadCount = 10; // set via CLI option; limiting write will limit output files @Override @@ -96,9 +98,9 @@ public class ExportApp extends ExportingToolBase { return options; } - + /** - * Tool entry point. + * Tool entry point. */ @Override public void runTool(CommandLine line) throws Exception { @@ -123,7 +125,7 @@ public class ExportApp extends ExportingToolBase { startSpring(); - UUID applicationId = emf.lookupApplication( applicationName ); + UUID applicationId = emf.lookupApplication( applicationName ).get(); if (applicationId == null) { throw new RuntimeException( "Cannot find application " + applicationName ); } @@ -134,63 +136,61 @@ public class ExportApp extends ExportingToolBase { writeScheduler = Schedulers.from( writeThreadPoolExecutor ); Observable<String> collectionsObservable = Observable.create( new CollectionsObservable( em ) ); - - collectionsObservable.flatMap( new Func1<String, Observable<ExportEntity>>() { - public Observable<ExportEntity> call(String collection) { + logger.debug( "Starting export" ); - return Observable.create( new EntityObservable( em, collection ) ) - .doOnNext( new EntityWriteAction() ).subscribeOn( writeScheduler ); - } + collectionsObservable.flatMap( collection -> { - }, writeThreadCount ).flatMap( new Func1<ExportEntity, Observable<ExportConnection>>() { + return Observable.create( new EntityObservable( em, collection ) ) + .doOnNext( new EntityWriteAction() ).subscribeOn( writeScheduler ); - public Observable<ExportConnection> call(ExportEntity exportEntity) { + } ).flatMap( exportEntity -> { - return Observable.create( new ConnectionsObservable( em, exportEntity ) ) - .doOnNext( new ConnectionWriteAction() ).subscribeOn( writeScheduler ); - } + return Observable.create( new ConnectionsObservable( em, exportEntity ) ) + .doOnNext( new ConnectionWriteAction() ).subscribeOn( writeScheduler ); - }, writeThreadCount ) - .doOnCompleted( new FileWrapUpAction() ) - .toBlocking().last(); + } ).doOnCompleted( new FileWrapUpAction() ).toBlocking().lastOrDefault(null); } - - + + // ---------------------------------------------------------------------------------------- // reading data - + /** * Emits collection names found in application. */ class CollectionsObservable implements rx.Observable.OnSubscribe<String> { EntityManager em; - + public CollectionsObservable(EntityManager em) { this.em = em; } public void call(Subscriber<? super String> subscriber) { - + int count = 0; try { Map<String, Object> collectionMetadata = em.getApplicationCollectionMetadata(); + + logger.debug( "Emitting {} collection names for application {}", + collectionMetadata.size(), em.getApplication().getName() ); + for ( String collection : collectionMetadata.keySet() ) { subscriber.onNext( collection ); count++; } - + } catch (Exception e) { subscriber.onError( e ); } - + subscriber.onCompleted(); logger.info( "Completed. Read {} collection names", count ); } } - + /** * Emits entities of collection. */ @@ -206,9 +206,9 @@ public class ExportApp extends ExportingToolBase { public void call(Subscriber<? super ExportEntity> subscriber) { logger.info("Starting to read entities of collection {}", collection); - - subscriber.onStart(); - + + //subscriber.onStart(); + try { int count = 0; @@ -229,16 +229,16 @@ public class ExportApp extends ExportingToolBase { } dictionariesByName.put( dictionary, dict ); } - - ExportEntity exportEntity = new ExportEntity( - organizationName, - applicationName, - entity, + + ExportEntity exportEntity = new ExportEntity( + organizationName, + applicationName, + entity, dictionariesByName ); - + subscriber.onNext( exportEntity ); count++; - + } catch (Exception e) { logger.error("Error reading entity " + entity.getUuid() +" from collection " + collection); } @@ -252,14 +252,14 @@ public class ExportApp extends ExportingToolBase { subscriber.onCompleted(); logger.info("Completed collection {}. Read {} entities", collection, count); - + } catch ( Exception e ) { subscriber.onError(e); } } } - + /** * Emits connections of an entity. */ @@ -274,53 +274,58 @@ public class ExportApp extends ExportingToolBase { public void call(Subscriber<? super ExportConnection> subscriber) { - logger.info( "Starting to read connections for entity {} type {}", - exportEntity.getEntity().getName(), exportEntity.getEntity().getType() ); - +// logger.debug( "Starting to read connections for entity {} type {}", +// exportEntity.getEntity().getName(), exportEntity.getEntity().getType() ); + int count = 0; - + try { Set<String> connectionTypes = em.getConnectionTypes( exportEntity.getEntity() ); for (String connectionType : connectionTypes) { - Results results = em.getConnectedEntities( - exportEntity.getEntity().getUuid(), connectionType, null, Results.Level.CORE_PROPERTIES ); + Results results = em.getTargetEntities( + exportEntity.getEntity(), connectionType, null, Query.Level.CORE_PROPERTIES ); for (Entity connectedEntity : results.getEntities()) { try { - - ExportConnection connection = new ExportConnection( + + ExportConnection connection = new ExportConnection( applicationName, organizationName, - connectionType, - exportEntity.getEntity().getUuid(), + connectionType, + exportEntity.getEntity().getUuid(), connectedEntity.getUuid()); - + subscriber.onNext( connection ); count++; } catch (Exception e) { - logger.error( "Error reading connection entity " + logger.error( "Error reading connection entity " + exportEntity.getEntity().getUuid() + " -> " + connectedEntity.getType()); } } } - + } catch (Exception e) { subscriber.onError( e ); } - + subscriber.onCompleted(); - logger.info("Completed entity {} type {} connections count {}", - new Object[] { exportEntity.getEntity().getName(), exportEntity.getEntity().getType(), count }); + + if ( count == 0 ) { + logger.debug("Completed entity {} type {} no connections", + new Object[] { exportEntity.getEntity().getUuid(), exportEntity.getEntity().getType() }); + } +// logger.debug("Completed entity {} type {} connections count {}", +// new Object[] { exportEntity.getEntity().getUuid(), exportEntity.getEntity().getType(), count }); } } - + // ---------------------------------------------------------------------------------------- // writing data - - + + /** * Writes entities to JSON file. */ @@ -358,7 +363,7 @@ public class ExportApp extends ExportingToolBase { } } - + /** * Writes connection to JSON file. */ @@ -396,7 +401,7 @@ public class ExportApp extends ExportingToolBase { } } - + private class FileWrapUpAction implements Action0 { @Override public void call() { http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java index 6fa4896..9c057ee 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java @@ -16,7 +16,6 @@ */ package org.apache.usergrid.tools; - import io.codearte.jfairy.Fairy; import io.codearte.jfairy.producer.company.Company; import io.codearte.jfairy.producer.person.Person; @@ -161,22 +160,30 @@ public class ExportDataCreator extends ToolBase { Entity userEntity = null; try { final Map<String, Object> userMap = new HashMap<String, Object>() {{ + put( "name", person.username() ); put( "username", person.username() ); put( "password", person.password() ); put( "email", person.email() ); put( "companyEmail", person.companyEmail() ); - put( "dateOfBirth", person.dateOfBirth() ); + put( "dateOfBirth", person.dateOfBirth().toDate().toString()); put( "firstName", person.firstName() ); put( "lastName", person.lastName() ); put( "nationalIdentificationNumber", person.nationalIdentificationNumber() ); put( "telephoneNumber", person.telephoneNumber() ); put( "passportNumber", person.passportNumber() ); - put( "address", person.getAddress() ); + put( "address", new HashMap<String, Object>() {{ + put("streetNumber", person.getAddress().streetNumber()); + put("street", person.getAddress().street()); + put("city", person.getAddress().getCity()); + put("postalCode", person.getAddress().getPostalCode()); + }}); }}; userEntity = em.create( "user", userMap ); users.add( userEntity ); + logger.debug("Created user {}", userEntity.getName()); + } catch (DuplicateUniquePropertyExistsException e) { logger.error( "Dup user generated: " + person.username() ); continue; @@ -185,11 +192,13 @@ public class ExportDataCreator extends ToolBase { continue; } + em.refreshIndex(); + final Company company = person.getCompany(); try { EntityRef ref = em.getAlias( "company", company.name() ); Entity companyEntity = (ref == null) ? null : em.get( ref ); - + // create company if it does not exist yet if ( companyEntity == null ) { final Map<String, Object> companyMap = new HashMap<String, Object>() {{ @@ -205,6 +214,7 @@ public class ExportDataCreator extends ToolBase { } em.createConnection( userEntity, "employer", companyEntity ); + logger.debug("User {} now employed by {}", userEntity.getName(), companyEntity.getName()); } catch (DuplicateUniquePropertyExistsException e) { logger.error( "Dup company generated {} property={}", company.name(), e.getPropertyName() ); @@ -213,7 +223,9 @@ public class ExportDataCreator extends ToolBase { logger.error("Error creating or connecting company", e); continue; } - + + em.refreshIndex(); + try { for (int j = 0; j < 5; j++) { Activity activity = new Activity(); @@ -225,24 +237,39 @@ public class ExportDataCreator extends ToolBase { activity.setContent( "User " + person.username() + " generated a random string " + RandomStringUtils.randomAlphanumeric( 5 ) ); em.createItemInCollection( userEntity, "activities", "activity", activity.getProperties() ); + + logger.debug("Created activity {}", activity.getContent()); } if (users.size() > 10) { for (int j = 0; j < 5; j++) { try { - em.createConnection( userEntity, "associate", users.get( (int) (Math.random() * users.size()) ) ); + Entity otherUser = users.get( (int) (Math.random() * users.size()) ); + em.createConnection( userEntity, "associate", otherUser ); + logger.debug("User {} now associated with user {}", + userEntity.getName(), otherUser.getName()); } catch (Exception e) { logger.error( "Error connecting user to user: " + e.getMessage() ); } } } - + + em.refreshIndex(); + + + Set<String> connectionTypes = em.getConnectionTypes( userEntity ); + + logger.debug("User {} now has {} connection types: {}", + new Object[] { userEntity.getName(), connectionTypes.size(), connectionTypes}); + } catch (Exception e) { logger.error("Error creating activities", e); continue; } } + + em.refreshIndex(); } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/Import.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Import.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Import.java index 01313e3..0861a0a 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/Import.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Import.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.UUID; +import com.google.common.base.Optional; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonParser; import org.codehaus.jackson.JsonToken; @@ -366,10 +367,10 @@ public class Import extends ToolBase { // Retrieve the namepsace for this collection. It's part of the name String applicationName = getApplicationFromColllection( collectionFileName ); - UUID appId = emf.lookupApplication( applicationName ); + Optional<UUID> appId = emf.lookupApplication( applicationName ); //no org in path, this is a pre public beta so we need to create the new path - if ( appId == null && !applicationName.contains( "/" ) ) { + if ( !appId.isPresent() && !applicationName.contains( "/" ) ) { String fileName = collectionFileName.replace( "collections", "application" ); File applicationFile = new File( importDir, fileName ); @@ -413,8 +414,8 @@ public class Import extends ToolBase { } - if ( appId == null ) { - logger.error( "Unable to find application with name {}. Skipping collections", appId ); + if ( !appId.isPresent() ) { + logger.error( "Unable to find application with name {}. Skipping collections", applicationName ); return; } @@ -427,7 +428,7 @@ public class Import extends ToolBase { jp.nextToken(); // START_OBJECT this is the outter hashmap - EntityManager em = emf.getEntityManager( appId ); + EntityManager em = emf.getEntityManager( appId.get() ); while ( jp.nextToken() != JsonToken.END_OBJECT ) { importEntitysStuff( jp, em );
