Repository: usergrid Updated Branches: refs/heads/export-feature ade6e7881 -> de6d58480
Heavy changes to how tests are going to be made and created in the future. Still trying to find out how to automate verification. Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bce8d21c Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bce8d21c Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bce8d21c Branch: refs/heads/export-feature Commit: bce8d21c82bcf60fb93020d26fd4c1ac4497c8b6 Parents: ade6e78 Author: George Reyes <g...@apache.org> Authored: Tue Oct 20 16:47:27 2015 -0700 Committer: George Reyes <g...@apache.org> Committed: Tue Oct 20 16:47:27 2015 -0700 ---------------------------------------------------------------------- .../management/export/MockS3ExportFilterIT.java | 164 ++++++++++++------- 1 file changed, 101 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/bce8d21c/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java index 3fe537d..667a36a 100644 --- a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java @@ -18,28 +18,35 @@ package org.apache.usergrid.management.export; import java.io.File; +import java.io.FileFilter; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.UUID; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.commons.io.filefilter.WildcardFileFilter; + import org.apache.usergrid.NewOrgAppAdminRule; import org.apache.usergrid.batch.JobExecution; import org.apache.usergrid.batch.service.JobSchedulerService; import org.apache.usergrid.management.OrganizationInfo; import org.apache.usergrid.management.UserInfo; import org.apache.usergrid.mq.Query; +import org.apache.usergrid.persistence.ConnectionRef; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; import org.apache.usergrid.persistence.SimpleEntityRef; @@ -53,6 +60,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.Service; +import it.unimi.dsi.fastutil.Hash; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -76,6 +85,8 @@ public class MockS3ExportFilterIT extends AbstractServiceIT { private UserInfo adminUser; private OrganizationInfo organization; private UUID applicationId; + private ExportService exportService = setup.getExportService(); + @Before public void setup() throws Exception { @@ -110,73 +121,61 @@ public class MockS3ExportFilterIT extends AbstractServiceIT { applicationId = newOrgAppAdminRule.getApplicationInfo().getId(); } + @After + public void after() { + File dir = new File("."); + FileFilter fileFilter = new WildcardFileFilter("entities*.json"); + File[] files = dir.listFiles(fileFilter); + for (File file : files) { + file.delete(); + } + + fileFilter = new WildcardFileFilter("connections*.json"); + files = dir.listFiles(fileFilter); + for (File file : files) { + file.delete(); + } + + } @Test //Connections won't save when run with maven, but on local builds it will. public void test1000ConnectionsToSingleEntity() throws Exception { + //setup String testFileName ="testConnectionsOnApplicationEndpoint.json"; - S3Export s3Export = new MockS3ExportImpl( testFileName ); - - ExportService exportService = setup.getExportService(); - String appName = newOrgAppAdminRule.getApplicationInfo().getName(); - HashMap<String, Object> payload = payloadBuilder( appName ); - - payload.put( "organizationId", organization.getUuid() ); - payload.put( "applicationId", applicationId ); - - EntityManager em = setup.getEmf().getEntityManager( applicationId ); // intialize user object to be posted - Map<String, Object> userProperties = null; - Entity[] entity; int numberOfEntitiesToBeWritten = 997; - entity = new Entity[numberOfEntitiesToBeWritten]; - - // creates entities - for ( int i = 0; i < numberOfEntitiesToBeWritten; i++ ) { - userProperties = new LinkedHashMap<String, Object>(); - userProperties.put( "username", "billybob" + i ); - userProperties.put( "email", "test" + i + "@anuff.com" ); - entity[i] = em.create( "users", userProperties ); - } + Entity[] entities = createEntities(numberOfEntitiesToBeWritten); for(int i = 1; i<numberOfEntitiesToBeWritten; i++){ - em.createConnection( em.get( new SimpleEntityRef( "user", entity[0].getUuid() ) ), "testConnections", - em.get( new SimpleEntityRef( "user", entity[i].getUuid() ) ) ); + createConnectionsBetweenEntities( entities[0],entities[i] ); } - setup.getEntityIndex().refresh( applicationId ); - - Thread.sleep( 1000 ); +// Create Payload to be sent to export job + Set applicationsToBeExported = new HashSet<>( ); + applicationsToBeExported.add( appName ); + HashMap<String, Object> payload = payloadBuilder( null,applicationsToBeExported,null,null ); - UUID exportUUID = exportService.schedule( payload ); - - //create and initialize jobData returned in JobExecution. - JobData jobData = jobDataCreator( payload, exportUUID, s3Export ); - - JobExecution jobExecution = mock( JobExecution.class ); - when( jobExecution.getJobData() ).thenReturn( jobData ); - - exportService.doExport( jobExecution ); + //Starts export. Setups up mocks for the job executor. + startExportJob( s3Export, payload ); TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() {}; - File exportedFile = new File("entities1"+testFileName); - exportedFile.delete(); - exportedFile = new File("entities2"+testFileName); - exportedFile.delete(); + Set<File> exportedConnectionFiles = returnsEntityFilesExported("connections" ,testFileName ); - final InputStream in = new FileInputStream( "connections1"+testFileName ); + final InputStream in = new FileInputStream( exportedConnectionFiles.iterator().next() ); try{ ObjectMapper mapper = new ObjectMapper(); JsonParser jp = new JsonFactory( ).createParser( in ); Iterator jsonIterator = mapper.readValues( jp, typeRef); HashMap jsonEntity = (HashMap)jsonIterator.next(); HashMap entityConnections = - ( HashMap ) ( jsonEntity ).get( entity[0].getUuid().toString() ); + ( HashMap ) ( jsonEntity ).get( entities[0].getUuid().toString() ); ArrayList connectionArray = (ArrayList)entityConnections.get( "testconnections" ); + //verifies that the number of connections should be equal to the number of entities written -1. assertEquals(numberOfEntitiesToBeWritten-1,connectionArray.size()); }catch(Exception e){ assertTrue(e.getMessage(),false ); @@ -185,14 +184,18 @@ public class MockS3ExportFilterIT extends AbstractServiceIT { finally{ in.close(); } + } - // Delete the created connection files - exportedFile = new File("connections1"+testFileName); - exportedFile.delete(); - exportedFile = new File("connections2"+testFileName); - exportedFile.delete(); + + private void startExportJob( final S3Export s3Export, final HashMap<String, Object> payload ) throws Exception {UUID + exportUUID = exportService.schedule( payload ); + JobData jobData = jobDataCreator( payload, exportUUID, s3Export ); + JobExecution jobExecution = mock( JobExecution.class ); + when( jobExecution.getJobData() ).thenReturn( jobData ); + exportService.doExport( jobExecution ); } + public JobData jobDataCreator( HashMap<String, Object> payload, UUID exportUUID, S3Export s3Export ) { JobData jobData = new JobData(); @@ -204,9 +207,7 @@ public class MockS3ExportFilterIT extends AbstractServiceIT { return jobData; } - - /*Creates fake payload for testing purposes.*/ - public HashMap<String, Object> payloadBuilder( String orgOrAppName ) { + public Map<String, Object> targetBuilder() { HashMap<String, Object> payload = new HashMap<String, Object>(); Map<String, Object> target = new HashMap<String, Object>(); Map<String, Object> storage_info = new HashMap<String, Object>(); @@ -221,19 +222,27 @@ public class MockS3ExportFilterIT extends AbstractServiceIT { return payload; } - public Map<String, Object> targetBuilder() { - HashMap<String, Object> payload = new HashMap<String, Object>(); - Map<String, Object> target = new HashMap<String, Object>(); - Map<String, Object> storage_info = new HashMap<String, Object>(); - storage_info.put( "s3_key", "null"); - storage_info.put( "s3_access_id", "null" ); - storage_info.put( "bucket_location", "null" ); + public Set returnsEntityFilesExported(String filenamePrefix,String filenameSuffix){ - target.put( "storage_provider", "s3" ); - target.put( "storage_info", storage_info ); + //keep reading files until there aren't any more to read. + int index = 1; + File exportedFile = new File( filenamePrefix + index + filenameSuffix ); - payload.put( "target", target ); - return payload; + Set<File> exportedEntityFiles = new HashSet<>( ); + + while(exportedFile.exists()) { + exportedEntityFiles.add( exportedFile ); + index++; + exportedFile = new File( filenamePrefix + index + filenameSuffix ); + } + + return exportedEntityFiles; + } + + public void deleteSetOfFiles(Set<File> filesExported){ + for(File exportedFile : filesExported){ + exportedFile.delete(); + } } @@ -270,14 +279,43 @@ public class MockS3ExportFilterIT extends AbstractServiceIT { } - public Map payloadBuilder(Query query,Set<String> applicationNames,Set<String> collectionNames, + public HashMap payloadBuilder(Query query,Set<String> applicationNames,Set<String> collectionNames, Set<String> connectionNames){ Map target = targetBuilder(); - Map filter = filterBuilder(query,applicationNames,collectionNames,connectionNames); + Map filters = filterBuilder(query,applicationNames,collectionNames,connectionNames); + HashMap payload = new HashMap<>( ); - + payload.put( "target",target ); + payload.put( "filters", filters ); + return payload; + } + + public Entity[] createEntities(final int numberOfEntitiesToBeCreated) throws Exception{ + EntityManager em = setup.getEmf().getEntityManager( applicationId ); + + // intialize user object to be posted + Map<String, Object> userProperties = null; + Entity[] entity; + entity = new Entity[numberOfEntitiesToBeCreated]; + + // creates entities + for ( int i = 0; i < numberOfEntitiesToBeCreated; i++ ) { + userProperties = new LinkedHashMap<String, Object>(); + userProperties.put( "username", "billybob" + i ); + userProperties.put( "email", "test" + i + "@anuff.com" ); + entity[i] = em.create( "users", userProperties ); + } + //refresh so entities appear immediately + setup.getEntityIndex().refresh( applicationId ); + + return entity; + } + + public ConnectionRef createConnectionsBetweenEntities (Entity entity1, Entity entity2) throws Exception{ + EntityManager em = setup.getEmf().getEntityManager( applicationId ); + return em.createConnection( em.get( new SimpleEntityRef( "user", entity1.getUuid() ) ), "testConnections", em.get( new SimpleEntityRef( "user", entity2.getUuid() ) ) ); } }
