Removing export and import files along with the other export that uses Jclouds
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c915a316 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c915a316 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c915a316 Branch: refs/heads/USERGRID-1246-MASTER Commit: c915a31683a2a93a1f124c958000723406aa4685 Parents: 5e40f42 Author: George Reyes <g...@apache.org> Authored: Fri Apr 29 10:48:11 2016 -0700 Committer: George Reyes <g...@apache.org> Committed: Mon May 2 10:50:49 2016 -0700 ---------------------------------------------------------------------- .../imports/FileErrorsResource.java | 140 -- .../imports/FileIncludesResource.java | 152 -- .../applications/imports/ImportsResource.java | 196 --- .../rest/management/ExportResourceIT.java | 594 -------- .../rest/management/ImportResourceIT.java | 795 ---------- .../usergrid/management/export/ExportJob.java | 89 -- .../management/export/ExportService.java | 49 - .../management/export/ExportServiceImpl.java | 634 -------- .../usergrid/management/export/S3Export.java | 26 - .../management/export/S3ExportImpl.java | 107 -- .../management/importer/FileImportJob.java | 124 -- .../management/importer/FileImportTracker.java | 319 ----- .../usergrid/management/importer/ImportJob.java | 118 -- .../management/importer/ImportService.java | 133 -- .../management/importer/ImportServiceImpl.java | 1352 ------------------ .../usergrid/management/importer/S3Import.java | 37 - .../management/importer/S3ImportImpl.java | 138 -- .../services/assets/data/S3BinaryStore.java | 346 ----- .../management/export/ExportServiceIT.java | 1047 -------------- .../management/export/MockS3ExportImpl.java | 56 - .../importer/FileImportTrackerTest.java | 399 ------ .../management/importer/ImportCollectionIT.java | 686 --------- .../importer/ImportConnectionsTest.java | 183 --- .../management/importer/ImportServiceIT.java | 678 --------- .../management/importer/MockS3ImportImpl.java | 42 - .../usergrid/management/importer/S3Upload.java | 117 -- 26 files changed, 8557 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c915a316/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileErrorsResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileErrorsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileErrorsResource.java deleted file mode 100644 index d9bc81b..0000000 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileErrorsResource.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.usergrid.rest.management.organizations.applications.imports; - - -import com.fasterxml.jackson.jaxrs.json.annotation.JSONP; -import org.apache.usergrid.management.ApplicationInfo; -import org.apache.usergrid.management.importer.ImportService; -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.Results; -import org.apache.usergrid.persistence.entities.FailedImportEntity; -import org.apache.usergrid.persistence.exceptions.EntityNotFoundException; -import org.apache.usergrid.rest.AbstractContextResource; -import org.apache.usergrid.rest.ApiResponse; -import org.apache.usergrid.rest.RootResource; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; - -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.PathSegment; -import javax.ws.rs.core.UriInfo; -import java.util.Collections; -import java.util.UUID; - - -@Component("org.apache.usergrid.rest.management.organizations.applications.imports.FileErrorsResource") -@Scope("prototype") -@Produces(MediaType.APPLICATION_JSON) -public class FileErrorsResource extends AbstractContextResource { - - - @Autowired - protected ImportService importService; - - private ApplicationInfo application; - private UUID importId; - private UUID importFileId; - - /** - * Override our service manager factory so that we get entities from the root management app - */ - public FileErrorsResource() { - //override the services management app - - } - - - public FileErrorsResource init( final ApplicationInfo application, final UUID importId, final UUID importFileId){ - this.application = application; - this.importId = importId; - this.importFileId = importFileId; - return this; - } - - - - @GET - @JSONP - @Produces({MediaType.APPLICATION_JSON, "application/javascript"}) - public ApiResponse getFileIncludes( @Context UriInfo ui, @QueryParam( "ql" ) String query, @QueryParam( "cursor" ) String cursor ) - throws Exception { - - - final Results importResults = importService.getFailedImportEntities( application.getId(), importId, - importFileId, query, cursor ); - - if(importResults == null){ - throw new EntityNotFoundException( "could not load import results" ); - } - - ApiResponse response = createApiResponse(); - - - response.setAction( "get" ); - response.setApplication( emf.getEntityManager( application.getId() ).getApplication() ); - response.setParams( ui.getQueryParameters() ); - - - response.withResults( importResults ); - - return response; - - } - - @GET - @Path( RootResource.ENTITY_ID_PATH ) - @JSONP - @Produces({MediaType.APPLICATION_JSON, "application/javascript"}) - public ApiResponse getFileIncludeById( @Context UriInfo ui, @PathParam( "entityId" ) PathSegment entityId ) - throws Exception { - - final UUID failedEntity = UUID.fromString( entityId.getPath() ); - final FailedImportEntity - importEntity = importService.getFailedImportEntity( application.getId(), importId, importFileId, - failedEntity ); - - if(importEntity == null){ - throw new EntityNotFoundException( "could not find import with uuid " + importId ); - } - - ApiResponse response = createApiResponse(); - - - response.setAction( "get" ); - response.setApplication( emf.getEntityManager( application.getId() ).getApplication() ); - response.setParams( ui.getQueryParameters() ); - - - response.setEntities( Collections.<Entity>singletonList( importEntity ) ); - - return response; - - } - - - - - - -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/c915a316/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileIncludesResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileIncludesResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileIncludesResource.java deleted file mode 100644 index 41ae84c..0000000 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileIncludesResource.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.usergrid.rest.management.organizations.applications.imports; - - -import java.util.Collections; -import java.util.UUID; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.PathSegment; -import javax.ws.rs.core.UriInfo; - -import com.fasterxml.jackson.jaxrs.json.annotation.JSONP; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; - -import org.apache.usergrid.management.ApplicationInfo; -import org.apache.usergrid.management.importer.ImportService; -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.Results; -import org.apache.usergrid.persistence.entities.FileImport; -import org.apache.usergrid.persistence.exceptions.EntityNotFoundException; -import org.apache.usergrid.rest.AbstractContextResource; -import org.apache.usergrid.rest.ApiResponse; -import org.apache.usergrid.rest.RootResource; - - -@Component("org.apache.usergrid.rest.management.organizations.applications.imports.FileIncludesResource") -@Scope("prototype") -@Produces(MediaType.APPLICATION_JSON) -public class FileIncludesResource extends AbstractContextResource { - - - @Autowired - protected ImportService importService; - - private ApplicationInfo application; - private UUID importId; - - /** - * Override our service manager factory so that we get entities from the root management app - */ - public FileIncludesResource() { - //override the services management app - - } - - - public FileIncludesResource init( final ApplicationInfo application, final UUID importId){ - this.application = application; - this.importId = importId; - return this; - } - - - - @GET - @JSONP - @Produces({MediaType.APPLICATION_JSON, "application/javascript"}) - public ApiResponse getFileIncludes( @Context UriInfo ui, @QueryParam( "ql" ) String query, @QueryParam( "cursor" ) String cursor ) - throws Exception { - - - final Results importResults = importService.getFileImports( application.getId(), importId, query, cursor ); - - if(importResults == null){ - throw new EntityNotFoundException( "could not load import results" ); - } - - ApiResponse response = createApiResponse(); - - - response.setAction( "get" ); - response.setApplication( emf.getEntityManager( application.getId() ).getApplication() ); - response.setParams( ui.getQueryParameters() ); - - - response.withResults( importResults ); - - return response; - - } - - @GET - @Path( RootResource.ENTITY_ID_PATH ) - @JSONP - @Produces({MediaType.APPLICATION_JSON, "application/javascript"}) - public ApiResponse getFileIncludeById( @Context UriInfo ui, @PathParam( "entityId" ) PathSegment entityId ) - throws Exception { - - final UUID fileIncludeId = UUID.fromString( entityId.getPath() ); - final FileImport importEntity = importService.getFileImport( application.getId(), importId, fileIncludeId ); - - if(importEntity == null){ - throw new EntityNotFoundException( "could not find import with uuid " + importId ); - } - - ApiResponse response = createApiResponse(); - - - response.setAction( "get" ); - response.setApplication( emf.getEntityManager( application.getId() ).getApplication() ); - response.setParams( ui.getQueryParameters() ); - - - response.setEntities( Collections.<Entity>singletonList( importEntity ) ); - - return response; - - } - - - - - - @Path( RootResource.ENTITY_ID_PATH + "/errors" ) - public FileErrorsResource getIncludes( @Context UriInfo ui, @PathParam( "entityId" ) PathSegment entityId ) - throws Exception { - - final UUID fileImportId = UUID.fromString( entityId.getPath() ); - return getSubResource( FileErrorsResource.class ).init( application, importId,fileImportId ); - - } - - - - -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/c915a316/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java deleted file mode 100644 index ab601fc..0000000 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.usergrid.rest.management.organizations.applications.imports; - - -import com.fasterxml.jackson.jaxrs.json.annotation.JSONP; -import org.apache.commons.lang.NullArgumentException; -import org.apache.usergrid.management.ApplicationInfo; -import org.apache.usergrid.management.OrganizationInfo; -import org.apache.usergrid.management.importer.ImportService; -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.Results; -import org.apache.usergrid.persistence.entities.Import; -import org.apache.usergrid.persistence.exceptions.EntityNotFoundException; -import org.apache.usergrid.rest.AbstractContextResource; -import org.apache.usergrid.rest.ApiResponse; -import org.apache.usergrid.rest.RootResource; -import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; - -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.PathSegment; -import javax.ws.rs.core.UriInfo; -import java.util.Collections; -import java.util.Map; -import java.util.UUID; - - -@Component( "org.apache.usergrid.rest.management.organizations.applications.imports.ImportsResource" ) -@Scope( "prototype" ) -@Produces( MediaType.APPLICATION_JSON ) -public class ImportsResource extends AbstractContextResource { - - private static final Logger logger = LoggerFactory.getLogger(ImportsResource.class); - - @Autowired - protected ImportService importService; - - private OrganizationInfo organization; - private ApplicationInfo application; - - - /** - * Override our service manager factory so that we get entities from the root management app - */ - public ImportsResource() { - //override the services management app - - } - - - public ImportsResource init( final OrganizationInfo organization, final ApplicationInfo application ) { - this.organization = organization; - this.application = application; - return this; - } - - - @POST - @RequireOrganizationAccess - @Consumes( MediaType.APPLICATION_JSON ) - @JSONP - @Produces({MediaType.APPLICATION_JSON, "application/javascript"}) - public ApiResponse executePost( @Context UriInfo ui, String body, - @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) - throws Exception { - - ApiResponse response = createApiResponse(); - response.setAction( "post" ); - response.setApplication( emf.getEntityManager( application.getId() ).getApplication() ); - response.setParams( ui.getQueryParameters() ); - - final Map<String, Object> json = ( Map<String, Object> ) readJsonToObject( body ); - - Map<String, Object> properties; - Map<String, Object> storage_info; - - if ( ( properties = ( Map<String, Object> ) json.get( "properties" ) ) == null ) { - throw new NullArgumentException( "Could not find 'properties'" ); - } - storage_info = ( Map<String, Object> ) properties.get( "storage_info" ); - String storage_provider = ( String ) properties.get( "storage_provider" ); - if ( storage_provider == null ) { - throw new NullArgumentException( "Could not find field 'storage_provider'" ); - } - if ( storage_info == null ) { - throw new NullArgumentException( "Could not find field 'storage_info'" ); - } - - String bucketName = ( String ) storage_info.get( "bucket_location" ); - - String accessId = (String) storage_info.get("s3_access_id"); - String secretKey = (String) storage_info.get("s3_key"); - - if (bucketName == null) { - throw new NullArgumentException("Could not find field 'bucketName'"); - } - if (accessId == null) { - throw new NullArgumentException("Could not find field 's3_access_id'"); - } - if (secretKey == null) { - - throw new NullArgumentException("Could not find field 's3_key'"); - } - - json.put( "organizationId", organization.getUuid() ); - json.put( "applicationId", application.getId() ); - - Import importEntity = importService.schedule( application.getId(), json ); - - response.setEntities( Collections.<Entity>singletonList( importEntity ) ); - - return response; - } - - - @GET - @JSONP - @Produces({MediaType.APPLICATION_JSON, "application/javascript"}) - public ApiResponse getImports( @Context UriInfo ui, @QueryParam( "ql" ) String query, @QueryParam( "cursor" ) String cursor ) throws Exception { - - final Results importResults = importService.getImports( application.getId(), query, cursor ); - - if ( importResults == null ) { - throw new EntityNotFoundException( "could not load import results" ); - } - - ApiResponse response = createApiResponse(); - response.setAction( "get" ); - response.setApplication( emf.getEntityManager( application.getId() ).getApplication() ); - response.setParams( ui.getQueryParameters() ); - response.withResults( importResults ); - - return response; - } - - - @GET - @Path( RootResource.ENTITY_ID_PATH ) - @JSONP - @Produces({MediaType.APPLICATION_JSON, "application/javascript"}) - public ApiResponse getImportById( @Context UriInfo ui, @PathParam( "entityId" ) PathSegment entityId ) - throws Exception { - - final UUID importId = UUID.fromString( entityId.getPath() ); - final Import importEntity = importService.getImport( application.getId(), importId ); - - if (logger.isTraceEnabled()) { - logger.trace("Loaded import entity {}:{} with state {}", - importEntity.getType(), importEntity.getUuid(), importEntity.getState()); - } - - if ( importEntity == null ) { - throw new EntityNotFoundException( "could not find import with uuid " + importId ); - } - - ApiResponse response = createApiResponse(); - response.setAction("get"); - response.setApplication(emf.getEntityManager(application.getId()).getApplication()); - response.setParams( ui.getQueryParameters() ); - response.setEntities( Collections.<Entity>singletonList( importEntity ) ); - - return response; - } - - - @Path( RootResource.ENTITY_ID_PATH + "/files" ) - public FileIncludesResource getIncludes( @Context UriInfo ui, @PathParam( "entityId" ) PathSegment entityId ) - throws Exception { - final UUID importId = UUID.fromString( entityId.getPath() ); - return getSubResource( FileIncludesResource.class ).init( application, importId ); - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/c915a316/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java deleted file mode 100644 index 81ff2d3..0000000 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java +++ /dev/null @@ -1,594 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.usergrid.rest.management; - - -import org.apache.usergrid.rest.test.resource.AbstractRestIT; -import org.apache.usergrid.rest.test.resource.model.ApiResponse; -import org.junit.Ignore; -import org.junit.Test; - -import javax.ws.rs.ClientErrorException; -import javax.ws.rs.core.Response; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import static org.junit.Assert.*; - - -@Ignore("Pending merge of export-feature branch. ") -public class ExportResourceIT extends AbstractRestIT { - - - - public ExportResourceIT() throws Exception { - - } - - @Test - public void exportApplicationUUIDRetTest() throws Exception { - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ).addToPath( "export" ); - } - catch ( ClientErrorException uie ) { - fail("We got back " + uie.getResponse().getStatus() + " instead of having a successful call" ); - } - - } - - @Test - public void exportCollectionUUIDRetTest() throws Exception { - - HashMap<String, Object> payload = payloadBuilder(); - ApiResponse exportEntity = null; - - try { - - - exportEntity = management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid()).addToPath( "collection" ) - .addToPath( "users" ).addToPath( "export" ).post(ApiResponse.class,payloadBuilder() ); - } - catch ( ClientErrorException uie ) { - fail( "We got back "+uie.getResponse().getStatus()+" instead of having a successful call" ); - } - - assertNotNull( exportEntity ); - assertNotNull( exportEntity.getProperties().get( "Export Entity" )); - } - - - /** - * Check that you can get the org export uuid returned and that you can check the status of the job using that uuid. - * @throws Exception - */ - @Test - public void exportGetOrganizationJobStatTest() throws Exception { - - ApiResponse exportEntity = null; - - - try { - exportEntity = management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).post( ApiResponse.class, payloadBuilder() ); - } - catch ( ClientErrorException uie ) { - fail( "We got back "+uie.getResponse().getStatus()+" instead of having a successful call" ); - } - - assertNotNull( exportEntity ); - String uuid = ( String ) exportEntity.getProperties().get( "Export Entity" ); - assertNotNull( uuid ); - - exportEntity = null; - try { - - exportEntity = management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).addToPath( uuid ).get( ApiResponse.class ); - } - catch ( ClientErrorException uie ) { - fail( "We got back "+uie.getResponse().getStatus()+" instead of having a successful call" ); - } - - assertNotNull( exportEntity ); - String state = (String) exportEntity.getProperties().get( "state" ); - assertEquals( "SCHEDULED", state); - } -// -// - - - /** - * Check that you can get the app export uuid returned and that you can check the status of the job using that uuid. - * @throws Exception - */ - - @Test - public void exportGetApplicationJobStatTest() throws Exception { - - ApiResponse exportEntity = null; - - - try { - exportEntity = management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "export" ).post( ApiResponse.class, payloadBuilder() ); - } - catch ( ClientErrorException uie ) { - fail( "We got back "+uie.getResponse().getStatus()+" instead of having a successful call" ); - } - - assertNotNull( exportEntity ); - String uuid = ( String ) exportEntity.getProperties().get( "Export Entity" ); - assertNotNull( uuid ); - - exportEntity = null; - refreshIndex(); - try { - - exportEntity = management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).addToPath( uuid ).get( ApiResponse.class ); - } - catch ( ClientErrorException uie ) { - fail( "We got back "+uie.getResponse().getStatus()+" instead of having a successful call" ); - } - - assertNotNull( exportEntity ); - String state = (String) exportEntity.getProperties().get( "state" ); - assertEquals( "SCHEDULED", state); - } - - - @Test - public void exportGetCollectionJobStatTest() throws Exception { - - ApiResponse exportEntity = null; - - exportEntity = management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid()).addToPath( "collection" ) - .addToPath( "users" ).addToPath( "export" ) - .post( ApiResponse.class, payloadBuilder() ); - - assertNotNull( exportEntity ); - String uuid = ( String ) exportEntity.getProperties().get( "Export Entity" ); - assertNotNull( uuid ); - - exportEntity = null; - try { - exportEntity = management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).addToPath( uuid ).get( ApiResponse.class ); - } - catch ( ClientErrorException uie ) { - fail( "We got back "+uie.getResponse().getStatus()+" instead of having a successful call" ); - } - - - assertNotNull( exportEntity ); - String state = (String) exportEntity.getProperties().get( "state" ); - assertEquals( "SCHEDULED", state); - } -// -// -// // //do an unauthorized test for both post and get -@Test - public void exportGetWrongUUID() throws Exception { - UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" ); - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).addToPath( fake.toString() ).get( ApiResponse.class ); - fail( "Should not have been able to get fake uuid" ); - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - - } - } - - - // - @Test - public void exportPostApplicationNullPointerProperties() throws Exception { - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "export" ).post( ApiResponse.class, - new HashMap<String, Object>() ); - fail( "Should not have passed, The payload is empty." ); - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - - @Test - public void exportPostOrganizationNullPointerProperties() throws Exception { - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).post( ApiResponse.class, new HashMap<String, Object>() ); - fail( "Should not have passed, The payload is empty." ); - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - // - @Test - public void exportPostCollectionNullPointer() throws Exception { - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "collection" ).addToPath( "users" ) - .addToPath( "export" ).post( ApiResponse.class, new HashMap<String, Object>() ); - - fail( "Should not have passed, The payload is empty." ); - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - @Test - public void exportGetCollectionUnauthorized() throws Exception { - UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "collection" ).addToPath( "users" ) - .addToPath( "export" ).addToPath( fake.toString() ).get(ApiResponse.class ,false); - fail( "Should not have passed as we didn't have an access token." ); - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.UNAUTHORIZED.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - @Test - public void exportGetApplicationUnauthorized() throws Exception { - UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "export" ).addToPath( fake.toString() ).get(ApiResponse.class ,false); - fail( "Should not have passed as we didn't have an access token." ); - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.UNAUTHORIZED.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - @Test - public void exportGetOrganizationUnauthorized() throws Exception { - UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).addToPath( fake.toString() ).get(ApiResponse.class ,false); - fail( "Should not have passed as we didn't have an access token." ); - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.UNAUTHORIZED.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - @Test - public void exportPostOrganizationNullPointerStorageInfo() throws Exception { - HashMap<String, Object> payload = payloadBuilder(); - HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" ); - //remove storage_info field - properties.remove( "storage_info" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - - @Test - public void exportPostApplicationNullPointerStorageInfo() throws Exception { - HashMap<String, Object> payload = payloadBuilder(); - HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" ); - //remove storage_info field - properties.remove( "storage_info" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "export" ).post( ApiResponse.class, - payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - @Test - public void exportPostCollectionNullPointerStorageInfo() throws Exception { - HashMap<String, Object> payload = payloadBuilder(); - HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" ); - //remove storage_info field - properties.remove( "storage_info" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "collection" ).addToPath( "users" ) - .addToPath( "export" ).post( ApiResponse.class, - payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - @Test - public void exportPostOrganizationNullPointerStorageProvider() throws Exception { - HashMap<String, Object> payload = payloadBuilder(); - HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" ); - //remove storage_info field - properties.remove( "storage_provider" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - - @Test - public void exportPostApplicationNullPointerStorageProvider() throws Exception { - HashMap<String, Object> payload = payloadBuilder(); - HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" ); - //remove storage_info field - properties.remove( "storage_provider" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "export" ).post( ApiResponse.class, - payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - @Test - public void exportPostCollectionNullPointerStorageProvider() throws Exception { - HashMap<String, Object> payload = payloadBuilder(); - HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" ); - //remove storage_info field - properties.remove( "storage_provider" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "collection" ).addToPath( "users" ) - .addToPath( "export" ).post( ApiResponse.class, - payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - - @Test - public void exportPostOrganizationNullPointerStorageVerification() throws Exception { - HashMap<String, Object> payload = payloadBuilder(); - HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" ); - HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" ); - //remove storage_key field - storage_info.remove( "s3_key" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - - payload = payloadBuilder(); - properties = ( HashMap<String, Object> ) payload.get( "properties" ); - storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" ); - //remove storage_key field - storage_info.remove( "s3_access_id"); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - - payload = payloadBuilder(); - properties = ( HashMap<String, Object> ) payload.get( "properties" ); - storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" ); - //remove storage_key field - storage_info.remove( "bucket_location" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - @Test - public void exportPostApplicationNullPointerStorageVerification() throws Exception { - HashMap<String, Object> payload = payloadBuilder(); - HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" ); - HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" ); - //remove storage_key field - storage_info.remove( "s3_key" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - - payload = payloadBuilder(); - properties = ( HashMap<String, Object> ) payload.get( "properties" ); - storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" ); - //remove storage_key field - storage_info.remove( "s3_access_id" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - - payload = payloadBuilder(); - properties = ( HashMap<String, Object> ) payload.get( "properties" ); - storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" ); - //remove storage_key field - storage_info.remove( "bucket_location" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - @Test - public void exportPostCollectionNullPointerStorageVerification() throws Exception { - HashMap<String, Object> payload = payloadBuilder(); - HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" ); - HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" ); - //remove storage_key field - storage_info.remove( "s3_key" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "collection" ).addToPath( "users" ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - - payload = payloadBuilder(); - properties = ( HashMap<String, Object> ) payload.get( "properties" ); - storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" ); - //remove storage_key field - storage_info.remove( "s3_access_id" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "collection" ).addToPath( "users" ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - - payload = payloadBuilder(); - properties = ( HashMap<String, Object> ) payload.get( "properties" ); - storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" ); - //remove storage_key field - storage_info.remove( "bucket_location" ); - - try { - management().orgs().org( clientSetup.getOrganizationName() ) - .app().addToPath( clientSetup.getAppUuid() ) - .addToPath( "collection" ).addToPath( "users" ) - .addToPath( "export" ).post( ApiResponse.class, payload ); - fail( "Should not have passed as we were missing an important part of the payload" ); - - } - catch ( ClientErrorException uie ) { - assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), uie.getResponse().getStatus() ); - } - } - - - /*Creates fake payload for testing purposes.*/ - public HashMap<String, Object> payloadBuilder() { - HashMap<String, Object> payload = new HashMap<String, Object>(); - Map<String, Object> properties = new HashMap<String, Object>(); - Map<String, Object> storage_info = new HashMap<String, Object>(); - //TODO: always put dummy values here and ignore this test. - //TODO: add a ret for when s3 values are invalid. - storage_info.put( "s3_key", "insert key here" ); - storage_info.put( "s3_access_id", "insert access id here" ); - storage_info.put( "bucket_location", "insert bucket name here" ); - properties.put( "storage_provider", "s3" ); - properties.put( "storage_info", storage_info ); - payload.put( "properties", properties ); - return payload; - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/c915a316/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java deleted file mode 100644 index c390393..0000000 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java +++ /dev/null @@ -1,795 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.usergrid.rest.management; - -import com.amazonaws.SDKGlobalConfiguration; -import com.google.common.collect.ImmutableSet; -import com.google.common.util.concurrent.Service; -import com.google.inject.Module; -import org.apache.commons.lang.RandomStringUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.usergrid.ServiceITSetup; -import org.apache.usergrid.ServiceITSetupImpl; -import org.apache.usergrid.batch.service.JobSchedulerService; -import org.apache.usergrid.management.importer.S3Upload; -import org.apache.usergrid.persistence.index.utils.UUIDUtils; -import org.apache.usergrid.rest.test.resource.AbstractRestIT; -import org.apache.usergrid.rest.test.resource.model.ApiResponse; -import org.apache.usergrid.rest.test.resource.model.Collection; -import org.apache.usergrid.rest.test.resource.model.Entity; -import org.apache.usergrid.rest.test.resource.model.Organization; -import org.apache.usergrid.rest.test.resource.model.Token; -import org.apache.usergrid.setup.ConcurrentProcessSingleton; - -import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.ContainerNotFoundException; -import org.jclouds.blobstore.domain.PageSet; -import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule; -import org.jclouds.logging.log4j.config.Log4JLoggingModule; -import org.jclouds.netty.config.NettyPayloadModule; -import org.junit.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.ws.rs.ClientErrorException; -import javax.ws.rs.core.Response; -import java.io.File; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Properties; - -import static org.junit.Assert.*; - - -@Ignore("Pending merge of export-feature branch. ") -public class ImportResourceIT extends AbstractRestIT { - - private static final Logger logger = LoggerFactory.getLogger(ImportResourceIT.class); - - - private static String bucketPrefix; - - private String bucketName; - - boolean configured; - - - public ImportResourceIT() throws Exception { - - } - - - @ClassRule - public static final ServiceITSetup setup = - new ServiceITSetupImpl(); - - @BeforeClass - public static void setup() throws Exception { - - bucketPrefix = System.getProperty("bucketName"); - - // start the scheduler after we're all set up - JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance() - .getSpringResource().getBean( JobSchedulerService.class ); - - if (jobScheduler.state() != Service.State.RUNNING) { - jobScheduler.startAsync(); - jobScheduler.awaitRunning(); - } - - } - - @Before - public void before() { - configured = - !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR )) - && !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR )) - && !StringUtils.isEmpty(System.getProperty("bucketName")); - - - if (!configured) { - logger.warn("Skipping test because {}, {} and bucketName not " + - "specified as system properties, e.g. in your Maven settings.xml file.", - new Object[]{ - "s3_key", - "s3_access_id" - }); - } - -// if (!StringUtils.isEmpty(bucketPrefix)) { -// deleteBucketsWithPrefix(); -// } - - bucketName = bucketPrefix + RandomStringUtils.randomAlphanumeric(10).toLowerCase(); - } - - - /** - * Verify that we can get call the import endpoint and get the job state back. - * - * @throws Exception - */ - @Test - public void importGetCollectionJobStatTest() throws Exception { - - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - Entity payload = payloadBuilder(); - - ///management/orgs/orgname/apps/appname/import - Entity entity = this.management() - .orgs() - .org( org ) - .app() - .addToPath(app) - .addToPath("imports") - .post(Entity.class,payload); - - assertNotNull(entity); - - entity = this.management() - .orgs() - .org( org ) - .app() - .addToPath(app) - .addToPath("imports") - .addToPath(entity.getUuid().toString()) - .get(); - - assertNotNull(entity.getAsString("state")); - } - - /** - * Verify that import job can only be read with an authorized token and cannot be read - * with an invalid/notAllowed token. - */ - @Test - public void importTokenAuthorizationTest() throws Exception { - - // this test should post one import job with one token, - // then try to read back the job with another token - - // create an import job - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - Entity payload = payloadBuilder(); - - // /management/orgs/orgname/apps/appname/import - Entity entity = this.management().orgs().org( org ).app() - .addToPath(app) - .addToPath("imports") - .post(Entity.class,payload); - - assertNotNull(entity); - - // test that you can access the organization using the currently set token. - this.management().orgs().org( org ).app().addToPath(app) - .addToPath("imports").addToPath(entity.getUuid().toString()).get(); - - //create a new org/app - String newOrgName = "org" + UUIDUtils.newTimeUUID(); - String newOrgUsername = "orgusername" + UUIDUtils.newTimeUUID(); - String newOrgEmail = UUIDUtils.newTimeUUID() + "@usergrid.com"; - String newOrgPassword = "password1"; - Organization orgPayload = new Organization( - newOrgName, newOrgUsername, newOrgEmail, newOrgName, newOrgPassword, null); - Organization orgCreatedResponse = clientSetup.getRestClient().management().orgs().post(orgPayload); - this.refreshIndex(); - assertNotNull(orgCreatedResponse); - - - //log into the new org/app and get a token - Token tokenPayload = new Token("password", newOrgUsername, newOrgPassword); - Token newOrgToken = clientSetup.getRestClient().management().token().post(Token.class,tokenPayload); - - //save the old token and set the newly issued token as current - context().setToken(newOrgToken); - - - //try to read with the new token, which should fail as unauthorized - try { - this.management().orgs().org( org ).app().addToPath(app) - .addToPath("imports").addToPath(entity.getUuid().toString()).get(); - fail("Should not be able to read import job with unauthorized token"); - } catch (ClientErrorException ex) { - errorParse(401, "unauthorized", ex); - } - - } - - - @Test - public void importPostApplicationNullPointerProperties() throws Exception { - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - Response.Status responseStatus = Response.Status.OK; - - Entity payload = new Entity(); - - try { - this.management().orgs().org( org ).app().addToPath(app).addToPath("imports").post(Entity.class,payload); - } catch (ClientErrorException uie) { - responseStatus = Response.Status.fromStatusCode( uie.getResponse().getStatus() ); - } - assertEquals(Response.Status.BAD_REQUEST, responseStatus); - } - - @Test - public void importPostApplicationNullPointerStorageInfo() throws Exception { - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - Response.Status responseStatus = Response.Status.OK; - - Entity payload = payloadBuilder(); - Entity properties = (Entity) payload.get("properties"); - //remove storage_info field - properties.remove("storage_info"); - - try { - this.management().orgs().org( org ).app().addToPath(app).addToPath("imports").post(Entity.class,payload); - } catch (ClientErrorException uie) { - responseStatus = Response.Status.fromStatusCode( uie.getResponse().getStatus() ); - } - assertEquals(Response.Status.BAD_REQUEST, responseStatus); - } - - - @Test - public void importPostApplicationNullPointerStorageProvider() throws Exception { - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - Response.Status responseStatus = Response.Status.OK; - - Entity payload = payloadBuilder(); - Entity properties = (Entity) payload.get("properties"); - //remove storage_info field - properties.remove("storage_provider"); - - - try { - this.management().orgs().org( org ).app().addToPath(app).addToPath("imports").post(Entity.class,payload); - } catch (ClientErrorException uie) { - responseStatus = Response.Status.fromStatusCode( uie.getResponse().getStatus() ); - } - assertEquals(Response.Status.BAD_REQUEST, responseStatus); - } - - - @Test - public void importPostApplicationNullPointerStorageVerification() throws Exception { - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - Response.Status responseStatus = Response.Status.OK; - - Entity payload = payloadBuilder(); - - Entity properties = (Entity) payload.get("properties"); - Entity storage_info = (Entity) properties.get("storage_info"); - //remove storage_key field - storage_info.remove("s3_key"); - - try { - this.management().orgs().org( org ).app().addToPath(app).addToPath("imports").post(Entity.class,payload); - } catch (ClientErrorException uie) { - responseStatus = Response.Status.fromStatusCode( uie.getResponse().getStatus() ); - } - assertEquals(Response.Status.BAD_REQUEST, responseStatus); - - payload = payloadBuilder(); - properties = (Entity) payload.get("properties"); - storage_info = (Entity) properties.get("storage_info"); - //remove storage_key field - storage_info.remove("s3_access_id"); - - try { - this.management().orgs().org( org ).app().addToPath(app).addToPath("imports").post(Entity.class,payload); - } catch (ClientErrorException uie) { - responseStatus = Response.Status.fromStatusCode( uie.getResponse().getStatus() ); - } - assertEquals(Response.Status.BAD_REQUEST, responseStatus); - - payload = payloadBuilder(); - properties = (Entity) payload.get("properties"); - storage_info = (Entity) properties.get("storage_info"); - //remove storage_key field - storage_info.remove("bucket_location"); - - try { - this.management().orgs().org( org ).app().addToPath(app).addToPath("imports").post(Entity.class,payload); - } catch (ClientErrorException uie) { - responseStatus = Response.Status.fromStatusCode( uie.getResponse().getStatus() ); - } - assertEquals(Response.Status.BAD_REQUEST, responseStatus); - } - -// @Test -// public void testExportImportCollection() throws Exception { -// Assume.assumeTrue( configured ); -// // create a collection of "thing" entities in the first application, export to S3 -// try { -// -// Map<UUID, org.apache.usergrid.persistence.Entity> thingsMap = new HashMap<>(); -// List<org.apache.usergrid.persistence.Entity> things = new ArrayList<>(); -// createTestEntities(emApp1, thingsMap, things, "thing"); -// -// deleteBucket(); -// exportCollection( emApp1, "things" ); -// -// // create new second application, import the data from S3 -// -// final UUID appId2 = setup.getMgmtSvc().createApplication( -// organization.getUuid(), "second").getId(); -// -// final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2); -// importCollection( emApp2, "things" ); -// -// -// // make sure that it worked -// -// logger.debug("\n\nCheck connections\n"); -// -// List<org.apache.usergrid.persistence.Entity> importedThings = emApp2.getCollection( -// appId2, "things", null, Query.Level.ALL_PROPERTIES).getEntities(); -// assertTrue( !importedThings.isEmpty() ); -// -// // two things have connections -// -// int conCount = 0; -// for ( org.apache.usergrid.persistence.Entity e : importedThings ) { -// Results r = emApp2.getTargetEntities( e, "related", null, Query.Level.IDS); -// List<ConnectionRef> connections = r.getConnections(); -// conCount += connections.size(); -// } -// assertEquals( 2, conCount ); -// -// logger.debug("\n\nCheck dictionaries\n"); -// -// // first two items have things in dictionary -// -// EntityRef entity0 = importedThings.get(0); -// Map connected0 = emApp2.getDictionaryAsMap(entity0, "connected_types"); -// Map connecting0 = emApp2.getDictionaryAsMap(entity0, "connected_types"); -// Assert.assertEquals( 1, connected0.size() ); -// Assert.assertEquals( 1, connecting0.size() ); -// -// EntityRef entity1 = importedThings.get(1); -// Map connected1 = emApp2.getDictionaryAsMap(entity1, "connected_types"); -// Map connecting1 = emApp2.getDictionaryAsMap(entity1, "connected_types"); -// Assert.assertEquals( 1, connected1.size() ); -// Assert.assertEquals( 1, connecting1.size() ); -// -// // the rest rest do not have connections -// -// EntityRef entity2 = importedThings.get(2); -// Map connected2 = emApp2.getDictionaryAsMap(entity2, "connected_types"); -// Map connecting2 = emApp2.getDictionaryAsMap(entity2, "connected_types"); -// Assert.assertEquals( 0, connected2.size() ); -// Assert.assertEquals( 0, connecting2.size() ); -// -// // if entities are deleted from app1, they still exist in app2 -// -// logger.debug("\n\nCheck dictionary\n"); -// for ( org.apache.usergrid.persistence.Entity importedThing : importedThings ) { -// emApp1.delete( importedThing ); -// } -// emApp1.refreshIndex(); -// emApp2.refreshIndex(); -// -// importedThings = emApp2.getCollection( -// appId2, "things", null, Query.Level.ALL_PROPERTIES).getEntities(); -// assertTrue( !importedThings.isEmpty() ); -// -// } finally { -// deleteBucket(); -// } -// } - - - /** - * TODO: Test that importing bad JSON will result in an informative error message. - */ - @Test - public void testImportGoodJson() throws Exception { - // import from a bad JSON file - Assume.assumeTrue(configured); - - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - - String basePath = System.getProperty("target.directory") - + File.separator + "test-classes" + File.separator; - - List<String> filenames = new ArrayList<>( 1 ); - filenames.add( basePath + "testimport-correct-testcol.1.json"); - - // create 10 applications each with collection of 10 things, export all to S3 - S3Upload s3Upload = new S3Upload(); - s3Upload.copyToS3( - System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ), - System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ), - bucketName, filenames); - - // import all those exports from S3 into the default test application - - Entity importEntity = importCollection(); - - Entity importGet = this.management().orgs().org( org ).app() - .addToPath(app) - .addToPath( "imports" ) - .addToPath(importEntity.getUuid().toString()) - .get(); - - refreshIndex(); - - Entity importGetIncludes = this.management().orgs().org( org ).app() - .addToPath(app) - .addToPath("imports") - .addToPath(importEntity.getUuid().toString()) - .addToPath("files") - .get(); - - ApiResponse importGetIncludesResponse = importGetIncludes.getResponse(); - - assertNotNull(importGet); - assertNotNull( importGetIncludes ); - assertEquals( 1,importGetIncludesResponse.getEntityCount()); - - - final Entity includesEntity = importGetIncludesResponse.getEntities().get( 0 ); - - assertTrue( includesEntity.getAsString("fileName").endsWith("testimport-correct-testcol.1.json")); - - assertEquals(1, includesEntity.get( "importedConnectionCount" )); - assertEquals(1, includesEntity.get( "importedEntityCount" )); - - assertEquals("FINISHED", importGet.get("state")); - assertEquals(1, importGet.get("fileCount")); - - Collection collection = this.app().collection("things").get(); - - assertNotNull(collection); - assertEquals(1, collection.getNumOfEntities()); - assertEquals("thing0", collection.getResponse().getEntities().get(0).get("name")); - - - //TODO: make sure it checks the actual imported entities. And the progress they have made. - - } - - /** - * TODO: Test that importing bad JSON will result in an informative error message. - */ - @Test - public void testImportOneGoodOneBad() throws Exception { - - // import from a bad JSON file - Assume.assumeTrue(configured); - - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - - String basePath = System.getProperty("target.directory") - + File.separator + "test-classes" + File.separator; - - List<String> filenames = new ArrayList<>( 2 ); - filenames.add( basePath + "testimport-correct-testcol.1.json"); - filenames.add( basePath + "testImport.testApplication.2.json"); - - // create 10 applications each with collection of 10 things, export all to S3 - S3Upload s3Upload = new S3Upload(); - s3Upload.copyToS3( - System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ), - System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ), - bucketName, filenames); - - // import all those exports from S3 into the default test application - - Entity importEntity = importCollection(); - - Entity importGet = this.management().orgs().org( org ).app() - .addToPath(app) - .addToPath("imports") - .addToPath(importEntity.getUuid().toString()).get(); - - assertNotNull(importGet); - - assertEquals("FAILED", importGet.get("state")); - assertEquals(2, importGet.get("fileCount")); - - Collection collection = this.app().collection("things").get(); - - assertNotNull(collection); - assertEquals(1, collection.getNumOfEntities()); - assertEquals("thing0", collection.getResponse().getEntities().get(0).get("name")); - } - - /** - * TODO: Test that importing bad JSON will result in an informative error message. - */ - @Test - public void testImportOneBadFile() throws Exception { - // import from a bad JSON file - Assume.assumeTrue(configured); - - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - - String basePath = System.getProperty("target.directory") - + File.separator + "test-classes" + File.separator; - - List<String> filenames = new ArrayList<>( 1 ); - filenames.add( basePath + "testimport-bad-json.json"); - - // create 10 applications each with collection of 10 things, export all to S3 - S3Upload s3Upload = new S3Upload(); - s3Upload.copyToS3( - System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ), - System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ), - bucketName, filenames); - - // import all those exports from S3 into the default test application - - Entity importEntity = importCollection(); - - Entity importGet = this.management().orgs().org( org ).app() - .addToPath(app) - .addToPath("imports" ) - .addToPath(importEntity.getUuid().toString() ) - .get(); - - assertNotNull(importGet); - - assertEquals("FAILED", importGet.get("state")); - assertEquals(1, importGet.get("fileCount")); - - Collection collection = this.app().collection("things").get(); - - assertNotNull(collection); - assertEquals(0, collection.getNumOfEntities()); - - - } - - - /** - * TODO: Test that importing bad JSON will result in an informative error message. - */ - @Test - public void testImportBadJson() throws Exception { - - // import from a bad JSON file - Assume.assumeTrue(configured); - - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - - String basePath = System.getProperty("target.directory") - + File.separator + "test-classes" + File.separator; - - List<String> filenames = new ArrayList<>( 1 ); - filenames.add( basePath + "testimport-bad-json-testapp.3.json"); - - // create 10 applications each with collection of 10 things, export all to S3 - S3Upload s3Upload = new S3Upload(); - s3Upload.copyToS3( - System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ), - System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ), - bucketName, filenames); - - // import all those exports from S3 into the default test application - - Entity importEntity = importCollection(); - - // we should now have 100 Entities in the default app - - Entity importGet = this.management().orgs().org( org ).app() - .addToPath( app ) - .addToPath("imports") - .addToPath( importEntity.getUuid().toString() ) - .get(); - - Entity importGetIncludes = this.management().orgs().org( org ).app() - .addToPath(app) - .addToPath("imports" ) - .addToPath(importEntity.getUuid().toString() ) - .addToPath("files" ) - .get(); - - assertNotNull(importGet); - - //TODO: needs better error checking - assertNotNull(importGetIncludes); - - // check that error message indicates JSON parsing error - } - - /** - * Call importService to import files from the configured S3 bucket. - */ - private Entity importCollection() throws Exception { - - String org = clientSetup.getOrganizationName(); - String app = clientSetup.getAppName(); - - logger.debug("\n\nImport into new app {}\n", app); - - Entity importPayload = new Entity(new HashMap<String, Object>() {{ - put("properties", new HashMap<String, Object>() {{ - put("storage_provider", "s3"); - put("storage_info", new HashMap<String, Object>() {{ - put("s3_key", - System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR )); - put("s3_access_id", - System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR )); - put("bucket_location", bucketName); - }}); - }}); - }}); - - Entity importEntity = this.management().orgs().org( org ).app() - .addToPath(app) - .addToPath("imports") - .post(Entity.class,importPayload); - - int maxRetries = 120; - int retries = 0; - - while (retries++ < maxRetries) { - - Entity importGet = this.management().orgs().org( org ).app() - .addToPath( app ) - .addToPath( "imports") - .addToPath( importEntity.getUuid().toString() ) - .get(); - - if (importGet.get("state").equals("FINISHED") || importGet.get( "state" ).equals( "FAILED" )) { - break; - } - - if (logger.isDebugEnabled()) { - logger.debug("Waiting for import..."); - } - Thread.sleep(1000); - } - - refreshIndex(); - - return importEntity; - } - - /** - * Create test entities of a specified type. - * First two entities are connected. - */ - private void createTestEntities() throws Exception { - - logger.debug("\n\nCreating users in application {}\n", - clientSetup.getAppName()); - - List<org.apache.usergrid.persistence.Entity> created = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - String name = "test" + i; - Entity payload = new Entity(); - payload.put("name", name); - payload.put("username", name); - payload.put("email", name + "@test.com"); - this.app().collection("users").post(payload); - - - } - - this.refreshIndex(); - -// // first two things are related to each other -// em.createConnection(new SimpleEntityRef(type, created.get(0).getUuid()), -// "related", new SimpleEntityRef(type, created.get(1).getUuid())); -// em.createConnection(new SimpleEntityRef(type, created.get(1).getUuid()), -// "related", new SimpleEntityRef(type, created.get(0).getUuid())); -// -// em.refreshIndex(); - } - - /** - * Delete the configured s3 bucket. - */ - public void deleteBucket() { - - logger.debug("\n\nDelete bucket\n"); - - String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ); - String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ); - - Properties overrides = new Properties(); - overrides.setProperty("s3" + ".identity", accessId); - overrides.setProperty("s3" + ".credential", secretKey); - - final Iterable<? extends Module> MODULES = ImmutableSet.of(new JavaUrlHttpCommandExecutorServiceModule(), - new Log4JLoggingModule(), new NettyPayloadModule()); - - BlobStoreContext context = - ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES) - .overrides(overrides ).buildView(BlobStoreContext.class); - - BlobStore blobStore = context.getBlobStore(); - blobStore.deleteContainer(bucketName); - } - - // might be handy if you need to clean up buckets - private static void deleteBucketsWithPrefix() { - - logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix); - - String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ); - String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ); - - Properties overrides = new Properties(); - overrides.setProperty("s3" + ".identity", accessId); - overrides.setProperty("s3" + ".credential", secretKey); - - final Iterable<? extends Module> MODULES = ImmutableSet - .of(new JavaUrlHttpCommandExecutorServiceModule(), - new Log4JLoggingModule(), - new NettyPayloadModule()); - - BlobStoreContext context = - ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES) - .overrides(overrides ).buildView(BlobStoreContext.class); - - BlobStore blobStore = context.getBlobStore(); - final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list(); - - for (Object o : blobStoreList.toArray()) { - StorageMetadata s = (StorageMetadata) o; - - if (s.getName().startsWith(bucketPrefix)) { - try { - blobStore.deleteContainer(s.getName()); - } catch (ContainerNotFoundException cnfe) { - logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe); - } - logger.debug("Deleted bucket {}", s.getName()); - } - } - } - - - /*Creates fake payload for testing purposes.*/ - public Entity payloadBuilder() { - - String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ); - String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ); - - Entity storage_info = new Entity(); - storage_info.put("s3_key", secretKey); - storage_info.put("s3_access_id", accessId); - storage_info.put("bucket_location", bucketName) ; - - Entity properties = new Entity(); - properties.put("storage_provider", "s3"); - properties.put("storage_info", storage_info); - - Entity payload = new Entity(); - payload.put("properties", properties); - - return payload; - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/c915a316/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java deleted file mode 100644 index 03c3353..0000000 --- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.management.export; - - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import org.apache.usergrid.batch.JobExecution; -import org.apache.usergrid.batch.job.OnlyOnceJob; -import org.apache.usergrid.persistence.entities.JobData; - - -/** - * Make an enum here, that contains the state info (look at Scott's - * code and emulate that to see what you can return in the JSON object). - */ -@Component("exportJob") -public class ExportJob extends OnlyOnceJob { - public static final String EXPORT_ID = "exportId"; - private static final Logger logger = LoggerFactory.getLogger( ExportJob.class ); - - @Autowired - ExportService exportService; - - public ExportJob() { - if (logger.isTraceEnabled()) { - logger.trace("ExportJob created"); - } - } - - - @Override - public void doJob( JobExecution jobExecution ) throws Exception { - logger.info( "execute ExportJob {}", jobExecution.getJobId().toString() ); - - JobData jobData = jobExecution.getJobData(); - if ( jobData == null ) { - logger.error( "jobData cannot be null" ); - return; - } - - jobExecution.heartbeat(); - try { - exportService.doExport( jobExecution ); - } - catch ( Exception e ) { - logger.error( "Export Service failed to complete job: {}", e.getMessage() ); - return; - } - - logger.info( "executed ExportJob process completed" ); - } - - - @Override - protected long getDelay( JobExecution jobExecution ) throws Exception { - //return arbitrary number - return 100; - } - - - @Autowired - public void setExportService( final ExportService exportService ) { - this.exportService = exportService; - } - - - @Override - public void dead( final JobExecution execution ) throws Exception { - //To change body of implemented methods use File | Settings | File Templates. - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/c915a316/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java deleted file mode 100644 index 5a14012..0000000 --- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.management.export; - - -import java.util.Map; -import java.util.UUID; - -import org.apache.usergrid.batch.JobExecution; - - -/** - * Performs all functions related to exporting - */ -public interface ExportService { - - /** - * Schedules the export to execute - */ - UUID schedule( Map<String,Object> json) throws Exception; - - - /** - * Perform the export to the external resource - */ - void doExport( JobExecution jobExecution ) throws Exception; - - /** - * Returns the current state of the service. - */ - String getState( UUID state ) throws Exception; - - String getErrorMessage( UUID appId, UUID state ) throws Exception; - -}
