Repository: usergrid Updated Branches: refs/heads/1.x 6e1f5ec5b -> a121f3796
Log org names on export Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4c927789 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4c927789 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4c927789 Branch: refs/heads/1.x Commit: 4c927789e81de65ad9b4e9958ac28af217482f8f Parents: 6e1f5ec Author: Dave Johnson <snoopd...@apache.org> Authored: Wed Jan 6 08:40:32 2016 -0500 Committer: Dave Johnson <snoopd...@apache.org> Committed: Wed Jan 6 08:40:32 2016 -0500 ---------------------------------------------------------------------- .../org/apache/usergrid/tools/ExportAdmins.java | 71 ++++++++++---------- .../org/apache/usergrid/tools/ImportAdmins.java | 10 ++- 2 files changed, 44 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/4c927789/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java index d5dd42c..57d87e6 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java @@ -46,44 +46,44 @@ import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEM * Export Admin Users and metadata including organizations and passwords. * * Usage Example: - * + * * java -Xmx8000m -Dlog4j.configuration=file:/home/me/log4j.properties -classpath . \ * -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 \ * -host casshost -inputDir=/home/me/export-data - * + * * If you want to provide any property overrides, put properties file named usergrid-custom-tools.properties * in the same directory where you run the above command. For example, you might want to set the Cassandra * client threads and export from a specific set of keyspaces: - * + * * cassandra.connections=110 * cassandra.system.keyspace=My_Usergrid * cassandra.application.keyspace=My_Usergrid_Applications * cassandra.lock.keyspace=My_Usergrid_Locks */ public class ExportAdmins extends ExportingToolBase { - + static final Logger logger = LoggerFactory.getLogger( ExportAdmins.class ); - + public static final String ADMIN_USERS_PREFIX = "admin-users"; public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata"; - + // map admin user UUID to list of organizations to which user belongs private Map<UUID, List<Org>> userToOrgsMap = new HashMap<UUID, List<Org>>(50000); private Map<String, UUID> orgNameToUUID = new HashMap<String, UUID>(50000); - + private Set<UUID> orgsWritten = new HashSet<UUID>(50000); - + private Set<UUID> duplicateOrgs = new HashSet<UUID>(); - + private static final String READ_THREAD_COUNT = "readThreads"; private int readThreadCount; AtomicInteger userCount = new AtomicInteger( 0 ); - + boolean ignoreInvalidUsers = false; // true to ignore users with no credentials or orgs - - + + /** * Represents an AdminUser that has been read and is ready for export. */ @@ -93,7 +93,7 @@ public class ExportAdmins extends ExportingToolBase { BiMap<UUID, String> orgNamesByUuid; } - + /** * Represents an organization associated with a user. */ @@ -138,7 +138,7 @@ public class ExportAdmins extends ExportingToolBase { } buildOrgMap(); - + // start write queue worker BlockingQueue<AdminUserWriteTask> writeQueue = new LinkedBlockingQueue<AdminUserWriteTask>(); @@ -180,7 +180,7 @@ public class ExportAdmins extends ExportingToolBase { } logger.debug( "Waiting for write thread to complete" ); - + boolean done = false; while ( !done ) { writeThread.join( 10000, 0 ); @@ -188,7 +188,7 @@ public class ExportAdmins extends ExportingToolBase { logger.info( "Wrote {} users", userCount.get() ); } } - + @Override @SuppressWarnings("static-access") @@ -225,7 +225,7 @@ public class ExportAdmins extends ExportingToolBase { execService.submit( new OrgMapWorker( organization ) ); count++; } - + if ( count % 1000 == 0 ) { logger.info("Queued {} org map workers", count); } @@ -237,24 +237,24 @@ public class ExportAdmins extends ExportingToolBase { while ( !execService.awaitTermination( 10, TimeUnit.SECONDS ) ) { logger.info( "Processed {} orgs for map", userToOrgsMap.size() ); } - + logger.info("Org map complete, counted {} organizations", count); } public class OrgMapWorker implements Runnable { private final Entity orgEntity; - + public OrgMapWorker( Entity orgEntity ) { this.orgEntity = orgEntity; } - + @Override public void run() { try { final String orgName = orgEntity.getProperty( "path" ).toString(); final UUID orgId = orgEntity.getUuid(); - + for (UserInfo user : managementService.getAdminUsersForOrganization( orgEntity.getUuid() )) { try { Entity admin = managementService.getAdminUserEntityByUuid( user.getUuid() ); @@ -340,17 +340,17 @@ public class ExportAdmins extends ExportingToolBase { if (ignoreInvalidUsers && (task.orgNamesByUuid.isEmpty() || task.dictionariesByName.isEmpty() || task.dictionariesByName.get( "credentials" ).isEmpty())) { - + actionTaken = "Ignored"; - + } else { writeQueue.add( task ); } - Map<String, Object> creds = (Map<String, Object>) (task.dictionariesByName.isEmpty() ? + Map<String, Object> creds = (Map<String, Object>) (task.dictionariesByName.isEmpty() ? 0 : task.dictionariesByName.get( "credentials" )); - - logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}", + + logger.warn( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}", new Object[]{ actionTaken, task.adminUser.getProperty( "username" ), @@ -359,7 +359,7 @@ public class ExportAdmins extends ExportingToolBase { task.orgNamesByUuid.size(), task.dictionariesByName.size(), creds == null ? 0 : creds.size() - } ); + } ); } catch ( Exception e ) { logger.error("Error reading data for user " + uuid, e ); @@ -370,13 +370,13 @@ public class ExportAdmins extends ExportingToolBase { private void addDictionariesToTask(AdminUserWriteTask task, Entity entity) throws Exception { EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID ); - + task.dictionariesByName = new HashMap<String, Map<Object, Object>>(); Set<String> dictionaries = em.getDictionaries( entity ); - + if ( dictionaries.isEmpty() ) { - logger.error("User {}:{} has no dictionaries", task.adminUser.getName(), task.adminUser.getUuid() ); + logger.warn("User {}:{} has no dictionaries", task.adminUser.getName(), task.adminUser.getUuid() ); return; } @@ -392,9 +392,9 @@ public class ExportAdmins extends ExportingToolBase { task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() ); List<Org> orgs = userToOrgsMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() ); - + if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) { - + // list of orgs from getOrganizationsForAdminUser() is less than expected, use userToOrgsMap BiMap<UUID, String> bimap = HashBiMap.create(); for (Org org : orgs) { @@ -455,9 +455,9 @@ public class ExportAdmins extends ExportingToolBase { saveOrganizations( metadataFile, task ); saveDictionaries( metadataFile, task ); - + metadataFile.writeEndObject(); - + logger.debug( "Exported user {}:{}:{}", new Object[] { task.adminUser.getProperty("username"), task.adminUser.getProperty("email"), @@ -527,8 +527,9 @@ public class ExportAdmins extends ExportingToolBase { jg.writeObject( orgs.get( uuid ) ); jg.writeEndObject(); - + synchronized (orgsWritten) { + logger.info("Exported org {}:{}", uuid, orgs.get(uuid)); orgsWritten.add( uuid ); } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/4c927789/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java index f39ef9b..cbd37dc 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java @@ -453,8 +453,14 @@ public class ImportAdmins extends ToolBase { managementService.createOrganization( orgUuid, orgName, userInfo, false ); orgInfo = managementService.getOrganizationByUuid( orgUuid ); - logger.debug( "Created new org {} for user {}", - new Object[]{orgInfo.getName(), user.getEmail()} ); + if ( orgInfo == null ) { + logger.error( "Error creating new org {} for user {}", + new Object[] { orgName, userInfo.getUsername() } ); + + } else { + logger.debug( "Created new org {} for user {}", + new Object[] { orgInfo.getName(), user.getEmail()} ); + } } catch (DuplicateUniquePropertyExistsException dpee) { logger.debug( "Org {} already exists", orgName );
