Add a new Asset binary provider for Google Cloud Storage. Refactor the BinaryStore implementation to follow a Factory pattern.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bafd4627 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bafd4627 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bafd4627 Branch: refs/heads/master Commit: bafd462744803f28175b13d0c205e3b029a6ac54 Parents: 949b465 Author: Michael Russo <russomich...@google.com> Authored: Wed Apr 26 15:43:40 2017 -0700 Committer: Michael Russo <russomich...@google.com> Committed: Wed Apr 26 15:43:40 2017 -0700 ---------------------------------------------------------------------- .../main/resources/usergrid-default.properties | 22 +- .../test/resources/usergrid-test-context.xml | 6 +- .../rest/applications/ServiceResource.java | 62 +-- .../applications/assets/AssetsResource.java | 38 +- .../main/resources/usergrid-rest-context.xml | 15 +- .../applications/assets/AssetResourceIT.java | 2 +- .../applications/assets/AwsAssetResourceIT.java | 4 +- .../assets/GoogleAssetResourceIT.java | 419 +++++++++++++++++++ .../applications/assets/aws/NoAWSCredsRule.java | 124 ------ .../assets/rules/NoAWSCredsRule.java | 136 ++++++ .../assets/rules/NoGoogleCredsRule.java | 118 ++++++ stack/services/pom.xml | 6 + .../services/assets/BinaryStoreFactory.java | 83 ++++ .../services/assets/data/AWSBinaryStore.java | 320 ++++++++++++++ .../assets/data/AwsSdkS3BinaryStore.java | 317 -------------- .../services/assets/data/GoogleBinaryStore.java | 223 ++++++++++ .../assets/data/LocalFileBinaryStore.java | 17 +- .../services/assets/data/S3BinaryStore.java | 346 --------------- .../test/resources/usergrid-test-context.xml | 12 +- 19 files changed, 1381 insertions(+), 889 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/config/src/main/resources/usergrid-default.properties ---------------------------------------------------------------------- diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties index 9c75ef0..0d1a193 100644 --- a/stack/config/src/main/resources/usergrid-default.properties +++ b/stack/config/src/main/resources/usergrid-default.properties @@ -565,13 +565,29 @@ usergrid.binary.bucketname=usergrid-binaries # usergrid.temp.files=/tmp/usergrid -# Set the implementation of binary uploading to be used -# Aws for aws s3 uploading, local +# Set the implementation of binary uploading to be used. Valid values: +# +# local - uses the local file system and "usergrid.temp.files" specifies the location +# aws - uses AWS Simple Storage Service (S3) +# google - uses Google Cloud Platform Storage service (GCS) +# +# +# If Google is specified, the following ENVIRONMENT variable must be set on the system running Usergrid: +# +# GOOGLE_APPLICATION_CREDENTIALS=/full/path/to/credentialfile.json +# +# See: https://developers.google.com/identity/protocols/application-default-credentials#howtheywork +# +# +# If AWS is specified, the following JVM Arguments must be set: +# +# AWS_ACCESS_KEY_ID=awsKeyId +# AWS_SECRET_KEY=awsKeySecret +# usergrid.binary.uploader=local - ############################### Usergrid Admin ############################## # # Usergrid has a sysadmin user which has access to the complete Usergrid system http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/query-validator/src/test/resources/usergrid-test-context.xml ---------------------------------------------------------------------- diff --git a/stack/query-validator/src/test/resources/usergrid-test-context.xml b/stack/query-validator/src/test/resources/usergrid-test-context.xml index 7affe6d..19af73d 100644 --- a/stack/query-validator/src/test/resources/usergrid-test-context.xml +++ b/stack/query-validator/src/test/resources/usergrid-test-context.xml @@ -44,7 +44,11 @@ </bean> - <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore"/> + <bean id="binaryStoreFactory" class="org.apache.usergrid.services.assets.BinaryStoreFactory"> + <constructor-arg name="properties" ref="properties"/> + <constructor-arg name="entityManagerFactory" ref="entityManagerFactory"/> + <constructor-arg name="reposLocation" value="${usergrid.temp.files}"/> + </bean> <bean id="setup" class="org.apache.usergrid.corepersistence.CpSetup"> <constructor-arg ref="entityManagerFactory"/> http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java index 3835b75..9373f5e 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java @@ -20,6 +20,7 @@ package org.apache.usergrid.rest.applications; import com.amazonaws.AmazonServiceException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.jaxrs.json.annotation.JSONP; +import com.google.cloud.storage.StorageException; import org.apache.commons.lang.StringUtils; import org.apache.usergrid.management.OrganizationConfig; import org.apache.usergrid.management.OrganizationConfigProps; @@ -34,10 +35,8 @@ import org.apache.usergrid.rest.applications.assets.AssetsResource; import org.apache.usergrid.rest.security.annotations.CheckPermissionsForPath; import org.apache.usergrid.security.oauth.AccessInfo; import org.apache.usergrid.services.*; -import org.apache.usergrid.services.assets.data.AssetUtils; -import org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore; -import org.apache.usergrid.services.assets.data.BinaryStore; -import org.apache.usergrid.services.assets.data.LocalFileBinaryStore; +import org.apache.usergrid.services.assets.BinaryStoreFactory; +import org.apache.usergrid.services.assets.data.*; import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException; import org.apache.usergrid.utils.JsonUtils; import org.glassfish.jersey.media.multipart.BodyPart; @@ -46,6 +45,7 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanInfoFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -70,15 +70,10 @@ public class ServiceResource extends AbstractContextResource { protected static final Logger logger = LoggerFactory.getLogger( ServiceResource.class ); private static final String FILE_FIELD_NAME = "file"; - - // @Autowired private BinaryStore binaryStore; @Autowired - private LocalFileBinaryStore localFileBinaryStore; - - @Autowired - private AwsSdkS3BinaryStore awsSdkS3BinaryStore; + private BinaryStoreFactory binaryStoreFactory; protected ServiceManager services; @@ -89,17 +84,6 @@ public class ServiceResource extends AbstractContextResource { } - public void setBinaryStore(String binaryStoreType){ - - //TODO:GREY change this to be a property held elsewhere - if(binaryStoreType.equals("local")){ - this.binaryStore = localFileBinaryStore; - } - else{ - this.binaryStore = awsSdkS3BinaryStore; - } - } - @Override public void setParent( AbstractContextResource parent ) { @@ -749,14 +733,8 @@ public class ServiceResource extends AbstractContextResource { @Produces({MediaType.APPLICATION_JSON, "application/javascript"}) private ApiResponse executeMultiPart( UriInfo ui, String callback, FormDataMultiPart multiPart, ServiceAction serviceAction ) throws Exception { - - //needed for testing - if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){ - this.binaryStore = localFileBinaryStore; - } - else{ - this.binaryStore = awsSdkS3BinaryStore; - } + // needed for testing + this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) ); // collect form data values List<BodyPart> bodyParts = multiPart.getBodyParts(); @@ -831,12 +809,8 @@ public class ServiceResource extends AbstractContextResource { public Response uploadDataStream( @Context UriInfo ui, InputStream uploadedInputStream ) throws Exception { //needed for testing - if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){ - this.binaryStore = localFileBinaryStore; - } - else{ - this.binaryStore = awsSdkS3BinaryStore; - } + this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) ); + ApiResponse response = createApiResponse(); response.setAction( "get" ); @@ -871,13 +845,8 @@ public class ServiceResource extends AbstractContextResource { logger.trace( "ServiceResource.executeStreamGet" ); } - //Needed for testing - if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){ - this.binaryStore = localFileBinaryStore; - } - else{ - this.binaryStore = awsSdkS3BinaryStore; - } + // needed for testing + this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) ); ApiResponse response = createApiResponse(); response.setAction( "get" ); @@ -945,7 +914,6 @@ public class ServiceResource extends AbstractContextResource { return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); } catch(AmazonServiceException ase){ - if( ase.getStatusCode() > 499 ){ logger.error(ase.getMessage()); }else if(logger.isDebugEnabled()){ @@ -953,6 +921,14 @@ public class ServiceResource extends AbstractContextResource { } return Response.status(ase.getStatusCode()).build(); } + catch (StorageException se){ + if( se.getCode() > 499 ){ + logger.error(se.getMessage()); + }else if(logger.isDebugEnabled()){ + logger.debug(se.getMessage()); + } + return Response.status(se.getCode()).build(); + } catch(RuntimeException re){ logger.error(re.getMessage()); return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java index b894205..f6fab41 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java @@ -26,10 +26,8 @@ import org.apache.usergrid.rest.ApiResponse; import org.apache.usergrid.rest.applications.ServiceResource; import org.apache.usergrid.rest.security.annotations.CheckPermissionsForPath; import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess; -import org.apache.usergrid.services.assets.data.AssetUtils; -import org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore; -import org.apache.usergrid.services.assets.data.BinaryStore; -import org.apache.usergrid.services.assets.data.LocalFileBinaryStore; +import org.apache.usergrid.services.assets.BinaryStoreFactory; +import org.apache.usergrid.services.assets.data.*; import org.apache.usergrid.utils.StringUtils; import org.glassfish.jersey.media.multipart.FormDataParam; import org.slf4j.Logger; @@ -56,15 +54,10 @@ public class AssetsResource extends ServiceResource { private static final Logger logger = LoggerFactory.getLogger( AssetsResource.class ); - //@Autowired private BinaryStore binaryStore; @Autowired - private LocalFileBinaryStore localFileBinaryStore; - - @Autowired - private AwsSdkS3BinaryStore awsSdkS3BinaryStore; - + private BinaryStoreFactory binaryStoreFactory; @Override @@ -123,12 +116,8 @@ public class AssetsResource extends ServiceResource { // @FormDataParam("file") FormDataContentDisposition fileDetail, @PathParam("entityId") PathSegment entityId ) throws Exception { - if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){ - this.binaryStore = localFileBinaryStore; - } - else{ - this.binaryStore = awsSdkS3BinaryStore; - } + // needed for testing + this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) ); if (uploadedInputStream != null ) { UUID assetId = UUID.fromString( entityId.getPath() ); @@ -162,12 +151,8 @@ public class AssetsResource extends ServiceResource { public Response uploadDataStream( @PathParam("entityId") PathSegment entityId, InputStream uploadedInputStream ) throws Exception { - if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){ - this.binaryStore = localFileBinaryStore; - } - else{ - this.binaryStore = awsSdkS3BinaryStore; - } + // needed for testing + this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) ); UUID assetId = UUID.fromString( entityId.getPath() ); if (logger.isTraceEnabled()) { @@ -191,12 +176,9 @@ public class AssetsResource extends ServiceResource { public Response findAsset( @Context UriInfo ui, @QueryParam("callback") @DefaultValue("callback") String callback, @PathParam("entityId") PathSegment entityId, @HeaderParam("range") String range, @HeaderParam("if-modified-since") String modifiedSince ) throws Exception { - if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){ - this.binaryStore = localFileBinaryStore; - } - else{ - this.binaryStore = awsSdkS3BinaryStore; - } + + // needed for testing + this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) ); UUID assetId = UUID.fromString( entityId.getPath() ); if (logger.isTraceEnabled()) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/main/resources/usergrid-rest-context.xml ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/resources/usergrid-rest-context.xml b/stack/rest/src/main/resources/usergrid-rest-context.xml index 5e925bf..2aaa93a 100644 --- a/stack/rest/src/main/resources/usergrid-rest-context.xml +++ b/stack/rest/src/main/resources/usergrid-rest-context.xml @@ -63,17 +63,10 @@ <property name="queueCapacity" value="25" /> </bean> - <bean id="serviceResource" - class="org.apache.usergrid.rest.applications.ServiceResource" scope="prototype"> - <property name="binaryStore" value="${usergrid.binary.uploader}"/> + <bean id="binaryStoreFactory" class="org.apache.usergrid.services.assets.BinaryStoreFactory"> + <constructor-arg name="properties" ref="properties"/> + <constructor-arg name="entityManagerFactory" ref="entityManagerFactory"/> + <constructor-arg name="reposLocation" value="${usergrid.temp.files}"/> </bean> - <bean id="localFileBinaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore"> - <property name="reposLocation" value="${usergrid.temp.files}"/> - </bean> - - <bean id="awsSdkS3BinaryStore" class="org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore"> - </bean> - - </beans> http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java index 616d929..cc9c326 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java @@ -281,7 +281,7 @@ public class AssetResourceIT extends AbstractRestIT { @Test - public void largeFileInS3() throws Exception { + public void largeFile() throws Exception { this.waitForQueueDrainAndRefreshIndex(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java index 4a9bfaa..196c662 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java @@ -21,7 +21,7 @@ import com.amazonaws.SDKGlobalConfiguration; import net.jcip.annotations.NotThreadSafe; import org.apache.commons.io.IOUtils; -import org.apache.usergrid.rest.applications.assets.aws.NoAWSCredsRule; +import org.apache.usergrid.rest.applications.assets.rules.NoAWSCredsRule; import org.apache.usergrid.rest.test.resource.AbstractRestIT; import org.apache.usergrid.rest.test.resource.model.ApiResponse; import org.apache.usergrid.rest.test.resource.model.Entity; @@ -58,7 +58,7 @@ public class AwsAssetResourceIT extends AbstractRestIT { * Mark tests as ignored if no AWS creds are present */ @Rule - public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule(); + public NoAWSCredsRule credsRule = new NoAWSCredsRule(); @Before public void setup(){ http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java new file mode 100644 index 0000000..811420e --- /dev/null +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java @@ -0,0 +1,419 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.rest.applications.assets; + + +import net.jcip.annotations.NotThreadSafe; +import org.apache.commons.io.IOUtils; +import org.apache.usergrid.rest.applications.assets.rules.NoAWSCredsRule; +import org.apache.usergrid.rest.applications.assets.rules.NoGoogleCredsRule; +import org.apache.usergrid.rest.test.resource.AbstractRestIT; +import org.apache.usergrid.rest.test.resource.model.ApiResponse; +import org.apache.usergrid.rest.test.resource.model.Entity; +import org.apache.usergrid.services.assets.data.AssetUtils; +import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.junit.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.ForbiddenException; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.NotFoundException; +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.UUID; + +import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER; +import static org.apache.usergrid.utils.MapUtils.hashMap; +import static org.junit.Assert.*; + + +@NotThreadSafe +public class GoogleAssetResourceIT extends AbstractRestIT { + + private Map<String, Object> originalProperties; + private static final Logger logger = LoggerFactory.getLogger(GoogleAssetResourceIT.class); + + /** + * Mark tests as ignored if no credentials are present + */ + @Rule + public NoGoogleCredsRule credsRule = new NoGoogleCredsRule(); + + @Before + public void setup() { + originalProperties = getRemoteTestProperties(); + setTestProperty(PROPERTIES_USERGRID_BINARY_UPLOADER, "google"); + + } + + @After + public void teardown() { + setTestProperties(originalProperties); + } + + + @Test + public void ensureMissingFileReturns404() { + Map<String, String> payload = hashMap("name", "assettest"); + ApiResponse postResponse = pathResource(getOrgAppPath("missingFile")).post(payload); + UUID assetId = postResponse.getEntities().get(0).getUuid(); + assertNotNull(assetId); + + try { + pathResource(getOrgAppPath("missingFile/assettest")).getAssetAsStream(true); + fail("Should fail as there isn't an asset to retrieve."); + } catch (NotFoundException nfe) { + } catch (Exception e) { + logger.error("Unexpected exception", e); + fail("Shouldn't return any other kind of exception"); + } + + } + + @Test + public void errorCheckingInvalidProperties() throws Exception { + Map<String, Object> errorTestProperties; + errorTestProperties = getRemoteTestProperties(); + setTestProperty("usergrid.binary.bucketname", "xxx"); + + try { + + Map<String, String> payload = hashMap("name", "assetname"); + ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(payload); + UUID assetId = postResponse.getEntities().get(0).getUuid(); + assertNotNull(assetId); + + // post a binary asset to that entity + + byte[] data = IOUtils.toByteArray(getClass().getResourceAsStream("/cassandra_eye.jpg")); + ApiResponse putResponse = + pathResource(getOrgAppPath("foos/" + assetId)).put(data, MediaType.APPLICATION_OCTET_STREAM_TYPE); + + + } catch (AwsPropertiesNotFoundException e) { + fail("Shouldn't interrupt runtime if access key isnt found."); + } catch (InternalServerErrorException uie) { + assertEquals(500, uie.getResponse().getStatus()); + } finally { + setTestProperties(errorTestProperties); + } + } + + @Test + public void errorCheckingInvalidPropertiesMultipartUpload() throws Exception { + Map<String, Object> errorTestProperties; + errorTestProperties = getRemoteTestProperties(); + //test that we fail gracefully if we have missing properties + setTestProperty("usergrid.binary.bucketname", "xxx"); + + try { + + byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/file-bigger-than-5M")); + FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE); + ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(form); + UUID assetId = postResponse.getEntities().get(0).getUuid(); + logger.info("Waiting for upload to finish..."); + Thread.sleep(5000); + + // check that entire file was uploaded + + ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class); + logger.info("Upload complete!"); + InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream(); + byte[] foundData = IOUtils.toByteArray(is); + assertEquals(data.length, foundData.length); + + // delete file + + pathResource(getOrgAppPath("foos/" + assetId)).delete(); + + + } catch (AwsPropertiesNotFoundException e) { + fail("Shouldn't interrupt runtime if access key isnt found."); + } catch (ForbiddenException fe) { + assertEquals(403, fe.getResponse().getStatus()); + } finally { + setTestProperties(errorTestProperties); + } + } + + @Test + public void octetStreamOnDynamicEntity() throws Exception { + + this.waitForQueueDrainAndRefreshIndex(); + + // post an asset entity + + Map<String, String> payload = hashMap("name", "assetname"); + ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(payload); + UUID assetId = postResponse.getEntities().get(0).getUuid(); + assertNotNull(assetId); + + // post a binary asset to that entity + + byte[] data = IOUtils.toByteArray(getClass().getResourceAsStream("/cassandra_eye.jpg")); + ApiResponse putResponse = pathResource(getOrgAppPath("foos/" + assetId)) + .put(data, MediaType.APPLICATION_OCTET_STREAM_TYPE); + + // check that the asset entity has asset metadata + + ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class); + Entity entity = getResponse.getEntities().get(0); + Map<String, Object> fileMetadata = (Map<String, Object>) entity.get("file-metadata"); + Assert.assertEquals("image/jpeg", fileMetadata.get("content-type")); + Assert.assertEquals(7979, fileMetadata.get("content-length")); + assertEquals(assetId, entity.getUuid()); + + // get binary asset by UUID + + InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream(); + byte[] foundData = IOUtils.toByteArray(is); + assertEquals(7979, foundData.length); + + // get binary asset by name + + is = pathResource(getOrgAppPath("foos/assetname")).getAssetAsStream(); + foundData = IOUtils.toByteArray(is); + assertEquals(7979, foundData.length); + } + + + @Test + public void multipartPostFormOnDynamicEntity() throws Exception { + + this.waitForQueueDrainAndRefreshIndex(); + + // post data larger than 5M + + byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/file-bigger-than-5M")); + FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE); + ApiResponse putResponse = pathResource(getOrgAppPath("foos")).post(form); + this.waitForQueueDrainAndRefreshIndex(); + + UUID assetId = putResponse.getEntities().get(0).getUuid(); + assertNotNull(assetId); + + // retry until upload complete and we can get the data + + int retries = 0; + boolean done = false; + byte[] foundData = new byte[0]; + while (!done && retries < 30) { + + try { + InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream(); + foundData = IOUtils.toByteArray(is); + done = true; + + } catch (Exception intentiallyIgnored) { + } + + Thread.sleep(1000); + retries++; + } + + // did we get expected number of bytes of data? + + assertEquals(5324800, foundData.length); + + pathResource(getOrgAppPath("foos/" + assetId)).delete(); + } + + + @Test + public void multipartPutFormOnDynamicEntity() throws Exception { + + this.waitForQueueDrainAndRefreshIndex(); + + // post an entity + + Map<String, String> payload = hashMap("foo", "bar"); + ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(payload); + UUID assetId = postResponse.getEntities().get(0).getUuid(); + assertNotNull(assetId); + + // post asset to that entity + + byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/cassandra_eye.jpg")); + FormDataMultiPart form = new FormDataMultiPart() + .field("foo", "bar2") + .field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE); + ApiResponse putResponse = pathResource(getOrgAppPath("foos/" + assetId)).put(form); + this.waitForQueueDrainAndRefreshIndex(); + + // get entity and check asset metadata + + ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class); + Entity entity = getResponse.getEntities().get(0); + Map<String, Object> fileMetadata = (Map<String, Object>) entity.get("file-metadata"); + long lastModified = Long.parseLong(fileMetadata.get(AssetUtils.LAST_MODIFIED).toString()); + + assertEquals(assetId, entity.getUuid()); + assertEquals("bar2", entity.get("foo")); + assertEquals("image/jpeg", fileMetadata.get(AssetUtils.CONTENT_TYPE)); + assertEquals(7979, fileMetadata.get(AssetUtils.CONTENT_LENGTH)); + + // get asset and check size + + InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream(); + byte[] foundData = IOUtils.toByteArray(is); + assertEquals(7979, foundData.length); + + // upload new asset to entity, then check that it was updated + + ApiResponse putResponse2 = pathResource(getOrgAppPath("foos/" + assetId)).put(form); + entity = putResponse2.getEntities().get(0); + fileMetadata = (Map<String, Object>) entity.get("file-metadata"); + long justModified = Long.parseLong(fileMetadata.get(AssetUtils.LAST_MODIFIED).toString()); + assertNotEquals(lastModified, justModified); + } + + + @Test + public void largeFile() throws Exception { + + this.waitForQueueDrainAndRefreshIndex(); + + // upload file larger than 5MB + + byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/file-bigger-than-5M")); + FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE); + ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(form); + UUID assetId = postResponse.getEntities().get(0).getUuid(); + logger.info("Waiting for upload to finish..."); + Thread.sleep(5000); + + // check that entire file was uploaded + + ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class); + logger.info("Upload complete!"); + InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream(); + byte[] foundData = IOUtils.toByteArray(is); + assertEquals(data.length, foundData.length); + + // delete file + + pathResource(getOrgAppPath("foos/" + assetId)).delete(); + } + + @Test + public void fileTooLargeShouldResultInError() throws Exception { + + this.waitForQueueDrainAndRefreshIndex(); + + // set max file size down to 5mb + setTestProperty("usergrid.binary.max-size-mb", "5"); + + try { + + // upload a file larger than 6mb + + byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/ship-larger-than-6mb.gif")); + FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE); + ApiResponse postResponse = pathResource(getOrgAppPath("bars")).post(form); + UUID assetId = postResponse.getEntities().get(0).getUuid(); + + String errorMessage = null; + logger.info("Waiting for upload to finish..."); + Thread.sleep(1000); + + // attempt to get asset entity, it should contain error + + waitForQueueDrainAndRefreshIndex(); + ApiResponse getResponse = pathResource(getOrgAppPath("bars/" + assetId)).get(ApiResponse.class); + Map<String, Object> fileMetadata = (Map<String, Object>) getResponse.getEntities().get(0).get("file-metadata"); + assertNotNull(fileMetadata); + assertNotNull(fileMetadata.get("error")); + assertTrue(fileMetadata.get("error").toString().startsWith("Asset size ")); + + } finally { + + // set max upload size back to default 25mb + setTestProperties(originalProperties); + } + } + + /** + * Deleting a connection to an asset should not delete the asset or the asset's data + */ + @Test + public void deleteConnectionToAsset() throws IOException { + + this.waitForQueueDrainAndRefreshIndex(); + + // create the entity that will be the asset, an image + + Map<String, String> payload = hashMap("name", "cassandra_eye.jpg"); + ApiResponse postReponse = pathResource(getOrgAppPath("foos")).post(payload); + final UUID uuid = postReponse.getEntities().get(0).getUuid(); + + // post image data to the asset entity + + byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/cassandra_eye.jpg")); + pathResource(getOrgAppPath("foos/" + uuid)).put(data, MediaType.APPLICATION_OCTET_STREAM_TYPE); + + // create an imagegallery entity + + Map<String, String> imageGalleryPayload = hashMap("name", "my image gallery"); + + ApiResponse postResponse2 = pathResource(getOrgAppPath("imagegalleries")).post(imageGalleryPayload); + UUID imageGalleryId = postResponse2.getEntities().get(0).getUuid(); + + // connect imagegallery to asset + + ApiResponse connectResponse = pathResource( + getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/" + uuid)).post(ApiResponse.class); + this.waitForQueueDrainAndRefreshIndex(); + + // verify connection from imagegallery to asset + + ApiResponse containsResponse = pathResource( + getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/")).get(ApiResponse.class); + assertEquals(uuid, containsResponse.getEntities().get(0).getUuid()); + + // delete the connection + + pathResource(getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/" + uuid)).delete(); + this.waitForQueueDrainAndRefreshIndex(); + + // verify that connection is gone + + ApiResponse listResponse = pathResource( + getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/")).get(ApiResponse.class); + assertEquals(0, listResponse.getEntityCount()); + + // asset should still be there + + ApiResponse getResponse2 = pathResource(getOrgAppPath("foos/" + uuid)).get(ApiResponse.class); + Entity entity = getResponse2.getEntities().get(0); + Map<String, Object> fileMetadata = (Map<String, Object>) entity.get("file-metadata"); + + Assert.assertEquals("image/jpeg", fileMetadata.get(AssetUtils.CONTENT_TYPE)); + Assert.assertEquals(7979, fileMetadata.get(AssetUtils.CONTENT_LENGTH)); + assertEquals(uuid, entity.getUuid()); + + // asset data should still be there + + InputStream assetIs = pathResource(getOrgAppPath("foos/" + uuid)).getAssetAsStream(); + byte[] foundData = IOUtils.toByteArray(assetIs); + assertEquals(7979, foundData.length); + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java deleted file mode 100644 index 40a6b45..0000000 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.usergrid.rest.applications.assets.aws; - - -import com.amazonaws.AmazonClientException; -import com.amazonaws.SDKGlobalConfiguration; -import org.apache.usergrid.rest.test.resource.AbstractRestIT; -import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException; -import org.junit.Assume; -import org.junit.internal.runners.model.MultipleFailureException; -import org.junit.rules.TestRule; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.Map; -import java.util.Properties; - - -/** - * Created in an attempt to mark no aws cred tests as ignored. Blocked by this issue - * https://github.com/junit-team/junit/issues/116 - * - * Until then, simply marks as passed, which is a bit dangerous - */ -public class NoAWSCredsRule extends AbstractRestIT implements TestRule { - - @Autowired - private Properties properties; - - - public Statement apply( final Statement base, final Description description ) { - return new Statement() { - @Override - public void evaluate() throws Throwable { - String accessId; - String secretKey; - String bucketName; - try { - Map<String,Object> properties = getRemoteTestProperties(); - //TODO: GREY change this so that it checks for the properties, then if it doesn't have them, mark the tests as ignored. - accessId = (String)System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ); - secretKey = (String)System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ); - bucketName =(String) properties.get( "usergrid.binary.bucketname" ); - - if(accessId==null||secretKey==null||bucketName==null){ - throw new AwsPropertiesNotFoundException( "Access Keys" ); - } - base.evaluate(); - - } - catch ( Throwable t ) { - - if ( !isMissingCredsException( t ) ) { - throw t; - } - - //do this so our test gets marked as ignored. Not pretty, but it works - Assume.assumeTrue( false ); - - - } - } - }; - } - - - private boolean isMissingCredsException( final Throwable t ) { - - if ( t instanceof AmazonClientException ) { - - final AmazonClientException ace = ( AmazonClientException ) t; - - if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains( - "could not get aws secret key from system properties" ) ) { - //swallow - return true; - } - } - - if( t instanceof AwsPropertiesNotFoundException ){ - return true; - } - - /** - * Handle the multiple failure junit trace - */ - if( t instanceof MultipleFailureException ){ - for(final Throwable failure : ((MultipleFailureException)t).getFailures()){ - final boolean isMissingCreds = isMissingCredsException( failure ); - - if(isMissingCreds){ - return true; - } - } - } - final Throwable cause = t.getCause(); - - if ( cause == null ) { - return false; - } - - - return isMissingCredsException( cause ); - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java new file mode 100644 index 0000000..8606e4e --- /dev/null +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.rest.applications.assets.rules; + + +import com.amazonaws.AmazonClientException; +import com.amazonaws.SDKGlobalConfiguration; +import org.apache.usergrid.rest.test.resource.AbstractRestIT; +import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException; +import org.junit.Assume; +import org.junit.internal.runners.model.MultipleFailureException; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.FileInputStream; +import java.util.Map; +import java.util.Properties; + + +/** + * Created in an attempt to mark no aws cred tests as ignored. Blocked by this issue + * https://github.com/junit-team/junit/issues/116 + * + * Until then, simply marks as passed, which is a bit dangerous + */ +public class NoAWSCredsRule extends AbstractRestIT implements TestRule { + + private static final Logger logger = LoggerFactory.getLogger( NoAWSCredsRule.class ); + + + private Properties properties; + + + public Statement apply( final Statement base, final Description description ) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + + + try { + + final Map<String,Object> properties = getRemoteTestProperties(); + final String bucketName = (String)properties.get( "usergrid.binary.bucketname" ); + + + //TODO: GREY change this so that it checks for the properties, then if it doesn't have them, mark the tests as ignored. + final String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ); + final String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ); + + if(accessId==null||secretKey==null||bucketName==null){ + throw new AwsPropertiesNotFoundException( "Access Keys" ); + } + + + + + + base.evaluate(); + + } + catch ( Throwable t ) { + + if ( !isMissingCredsException( t ) ) { + throw t; + } + + //do this so our test gets marked as ignored. Not pretty, but it works + Assume.assumeTrue( false ); + + + } + } + }; + } + + + private boolean isMissingCredsException( final Throwable t ) { + + if ( t instanceof AmazonClientException ) { + + final AmazonClientException ace = ( AmazonClientException ) t; + + if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains( + "could not get aws secret key from system properties" ) ) { + //swallow + return true; + } + } + + if( t instanceof AwsPropertiesNotFoundException ){ + return true; + } + + /** + * Handle the multiple failure junit trace + */ + if( t instanceof MultipleFailureException ){ + for(final Throwable failure : ((MultipleFailureException)t).getFailures()){ + final boolean isMissingCreds = isMissingCredsException( failure ); + + if(isMissingCreds){ + return true; + } + } + } + final Throwable cause = t.getCause(); + + if ( cause == null ) { + return false; + } + + + return isMissingCredsException( cause ); + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java new file mode 100644 index 0000000..5e3f66e --- /dev/null +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.rest.applications.assets.rules; + + +import com.amazonaws.AmazonClientException; +import com.amazonaws.SDKGlobalConfiguration; +import org.apache.usergrid.rest.test.resource.AbstractRestIT; +import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException; +import org.junit.Assume; +import org.junit.internal.runners.model.MultipleFailureException; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.Map; +import java.util.Properties; + + +/** + * Created in an attempt to mark no Google cred tests as ignored. Blocked by this issue + * https://github.com/junit-team/junit/issues/116 + * + * Until then, simply marks as passed, which is a bit dangerous + */ +public class NoGoogleCredsRule extends AbstractRestIT implements TestRule { + + private static final Logger logger = LoggerFactory.getLogger( NoGoogleCredsRule.class ); + + + @Autowired + private Properties properties; + + + public Statement apply( final Statement base, final Description description ) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + + + try { + + final String filename = System.getenv( "GOOGLE_APPLICATION_CREDENTIALS" ); + logger.info("cred filename: {}", filename); + // if the file doesn't exist, an exception will be thrown + new FileInputStream(filename); + + base.evaluate(); + + } + catch ( Throwable t ) { + + if ( !isMissingCredsException( t ) ) { + throw t; + } + + //do this so our test gets marked as ignored. Not pretty, but it works + Assume.assumeTrue( false ); + + + } + } + }; + } + + + private boolean isMissingCredsException( final Throwable t ) { + + // either no filename was provided or the filename provided doesn't actually exist on the file system + if ( t instanceof FileNotFoundException || t instanceof NullPointerException ) { + return true; + } + + + /** + * Handle the multiple failure junit trace + */ + if( t instanceof MultipleFailureException ){ + for(final Throwable failure : ((MultipleFailureException)t).getFailures()){ + final boolean isMissingCreds = isMissingCredsException( failure ); + + if(isMissingCreds){ + return true; + } + } + } + final Throwable cause = t.getCause(); + + if ( cause == null ) { + return false; + } + + + return isMissingCredsException( cause ); + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/pom.xml ---------------------------------------------------------------------- diff --git a/stack/services/pom.xml b/stack/services/pom.xml index 8cd1756..b1df1b4 100644 --- a/stack/services/pom.xml +++ b/stack/services/pom.xml @@ -267,6 +267,12 @@ </dependency> <dependency> + <groupId>com.google.cloud</groupId> + <artifactId>google-cloud-storage</artifactId> + <version>0.11.0-beta</version> + </dependency> + + <dependency> <groupId>org.apache.jclouds</groupId> <artifactId>jclouds-blobstore</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java new file mode 100644 index 0000000..bd91dba --- /dev/null +++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.services.assets; + +import org.apache.usergrid.persistence.EntityManagerFactory; +import org.apache.usergrid.services.assets.data.AWSBinaryStore; +import org.apache.usergrid.services.assets.data.BinaryStore; +import org.apache.usergrid.services.assets.data.GoogleBinaryStore; +import org.apache.usergrid.services.assets.data.LocalFileBinaryStore; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.Properties; + +import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER; + + +public class BinaryStoreFactory { + + public enum Provider{ + local,aws,google + } + + private EntityManagerFactory entityManagerFactory; + private Properties properties; + private String reposLocation; + private LocalFileBinaryStore localFileBinaryStore; + private AWSBinaryStore awsBinaryStore; + private GoogleBinaryStore googleCloudStorageBinaryStore; + + public BinaryStoreFactory(Properties properties, EntityManagerFactory entityManagerFactory, String reposLocation) throws IOException, GeneralSecurityException { + this.properties = properties; + this.entityManagerFactory = entityManagerFactory; + this.reposLocation = reposLocation; + this.localFileBinaryStore = new LocalFileBinaryStore(properties, entityManagerFactory, reposLocation); + this.awsBinaryStore = new AWSBinaryStore(properties, entityManagerFactory, reposLocation); + this.googleCloudStorageBinaryStore = new GoogleBinaryStore(properties, entityManagerFactory, reposLocation); + } + + public synchronized BinaryStore getBinaryStore(String provider) throws IOException, GeneralSecurityException { + + provider = provider != null? provider.toLowerCase(): ""; + + if( provider.isEmpty() ){ + + if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( Provider.local.name() )){ + return localFileBinaryStore; + } else if (properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( Provider.google.name() )){ + return googleCloudStorageBinaryStore; + } else{ + return awsBinaryStore; + } + } + + if ( provider.equals(Provider.local.name())){ + return localFileBinaryStore; + } + if ( provider.equals(Provider.google.name())){ + return googleCloudStorageBinaryStore; + } + if( provider.equals(Provider.aws.name())){ + return awsBinaryStore; + } + + // this for backwards compatibility because historically anything other than "local" meant AWS + return awsBinaryStore; + } + +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java new file mode 100644 index 0000000..14da2e1 --- /dev/null +++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.services.assets.data; + + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.usergrid.persistence.Entity; +import org.apache.usergrid.persistence.EntityManager; +import org.apache.usergrid.persistence.EntityManagerFactory; +import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider; +import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException; +import org.apache.usergrid.utils.StringUtils; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; +import com.amazonaws.services.s3.model.DeleteObjectRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; +import com.amazonaws.services.s3.model.MultipartUpload; +import com.amazonaws.services.s3.model.MultipartUploadListing; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PartETag; +import com.amazonaws.services.s3.model.PutObjectResult; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.google.common.primitives.Ints; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.PushbackInputStream; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.codec.binary.Base64; + + +public class AWSBinaryStore implements BinaryStore { + + private static final Logger logger = LoggerFactory.getLogger(AWSBinaryStore.class ); + private static final long FIVE_MB = ( FileUtils.ONE_MB * 5 ); + + private AmazonS3 s3Client; + private String accessId; + private String secretKey; + private String bucketName; + private String regionName; + + private EntityManagerFactory emf; + private Properties properties; + private String reposLocation; + + public AWSBinaryStore(Properties properties, + EntityManagerFactory entityManagerFactory, + String reposLocation) { + this.properties = properties; + this.emf = entityManagerFactory; + this.reposLocation = reposLocation; + + } + + //TODO: GREY rework how the s3 client works because currently it handles initlization and returning of the client + //ideally it should only do one. and the client should be initlized at the beginning of the run. + private AmazonS3 getS3Client() throws Exception{ + + this.bucketName = properties.getProperty( "usergrid.binary.bucketname" ); + if(bucketName == null){ + logger.error( "usergrid.binary.bucketname not properly set so amazon bucket is null" ); + throw new AwsPropertiesNotFoundException( "usergrid.binary.bucketname" ); + + } + + final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider(); + AWSCredentials credentials = ugProvider.getCredentials(); + ClientConfiguration clientConfig = new ClientConfiguration(); + clientConfig.setProtocol(Protocol.HTTP); + + s3Client = new AmazonS3Client(credentials, clientConfig); + if(regionName != null) + s3Client.setRegion( Region.getRegion(Regions.fromName(regionName)) ); + + return s3Client; + } + + @Override + public void write( final UUID appId, final Entity entity, InputStream inputStream ) throws Exception { + + String uploadFileName = AssetUtils.buildAssetKey( appId, entity ); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB ); + + byte[] data = baos.toByteArray(); + + InputStream awsInputStream = new ByteArrayInputStream(data); + + final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity ); + fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() ); + + String mimeType = AssetMimeHandler.get().getMimeType( entity, data ); + + Boolean overSizeLimit = false; + + EntityManager em = emf.getEntityManager( appId ); + + + if ( written < FIVE_MB ) { // total smaller than 5mb + + ObjectMetadata om = new ObjectMetadata(); + om.setContentLength(written); + om.setContentType( mimeType ); + PutObjectResult result = null; + result = getS3Client().putObject( bucketName, uploadFileName, awsInputStream, om ); + + String md5sum = Hex.encodeHexString( Base64.decodeBase64(result.getContentMd5()) ); + String eTag = result.getETag(); + + fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); + + if(md5sum != null) + fileMetadata.put( AssetUtils.CHECKSUM, md5sum ); + fileMetadata.put( AssetUtils.E_TAG, eTag ); + + em.update( entity ); + + } + else { // bigger than 5mb... dump 5 mb tmp files and upload from them + written = 0; //reset written to 0, we still haven't wrote anything in fact + int partNumber = 1; + int firstByte = 0; + Boolean isFirstChunck = true; + List<PartETag> partETags = new ArrayList<PartETag>(); + + + //get the s3 client in order to initialize the multipart request + getS3Client(); + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest( bucketName, uploadFileName ); + InitiateMultipartUploadResult initResponse = getS3Client().initiateMultipartUpload( initRequest ); + + + InputStream firstChunck = new ByteArrayInputStream(data); + PushbackInputStream chunckableInputStream = new PushbackInputStream(inputStream, 1); + + // determine max size file allowed, default to 50mb + long maxSizeBytes = 50 * FileUtils.ONE_MB; + String maxSizeMbString = properties.getProperty( "usergrid.binary.max-size-mb", "50" ); + if ( StringUtils.isNumeric( maxSizeMbString )) { + maxSizeBytes = Long.parseLong( maxSizeMbString ) * FileUtils.ONE_MB; + } + + // always allow files up to 5mb + if (maxSizeBytes < 5 * FileUtils.ONE_MB ) { + maxSizeBytes = 5 * FileUtils.ONE_MB; + } + + while (-1 != (firstByte = chunckableInputStream.read())) { + long partSize = 0; + chunckableInputStream.unread(firstByte); + File tempFile = File.createTempFile( entity.getUuid().toString().concat("-part").concat(String.valueOf(partNumber)), "tmp" ); + + tempFile.deleteOnExit(); + OutputStream os = null; + try { + os = new BufferedOutputStream( new FileOutputStream( tempFile.getAbsolutePath() ) ); + + if(isFirstChunck == true) { + partSize = IOUtils.copyLarge( firstChunck, os, 0, ( FIVE_MB ) ); + isFirstChunck = false; + } + else { + partSize = IOUtils.copyLarge( chunckableInputStream, os, 0, ( FIVE_MB ) ); + } + written += partSize; + + if(written> maxSizeBytes){ + overSizeLimit = true; + logger.error( "OVERSIZED FILE ({}). STARTING ABORT", written ); + break; + //set flag here and break out of loop to run abort + } + } + finally { + IOUtils.closeQuietly( os ); + } + + FileInputStream chunk = new FileInputStream(tempFile); + + Boolean isLastPart = -1 == (firstByte = chunckableInputStream.read()); + if(!isLastPart) + chunckableInputStream.unread(firstByte); + + UploadPartRequest uploadRequest = new UploadPartRequest().withUploadId(initResponse.getUploadId()) + .withBucketName(bucketName) + .withKey(uploadFileName) + .withInputStream(chunk) + .withPartNumber(partNumber) + .withPartSize(partSize) + .withLastPart(isLastPart); + partETags.add( getS3Client().uploadPart(uploadRequest).getPartETag() ); + partNumber++; + } + + //check for flag here then abort. + if(overSizeLimit) { + + AbortMultipartUploadRequest abortRequest = + new AbortMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId() ); + + ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest( bucketName ); + + MultipartUploadListing listResult = getS3Client().listMultipartUploads( listRequest ); + + //upadte the entity with the error. + try { + logger.error("starting update of entity due to oversized asset"); + fileMetadata.put( "error", "Asset size is larger than max size of " + maxSizeBytes ); + em.update( entity ); + } + catch ( Exception e ) { + logger.error( "Error updating entity with error message", e ); + } + + int timesIterated = 20; + //loop and abort all the multipart uploads + while ( listResult.getMultipartUploads().size()!=0 && timesIterated > 0) { + + getS3Client().abortMultipartUpload( abortRequest ); + Thread.sleep( 1000 ); + timesIterated--; + listResult = getS3Client().listMultipartUploads( listRequest ); + if (logger.isDebugEnabled()) { + logger.debug("Files that haven't been aborted are: {}", listResult.getMultipartUploads().listIterator().toString()); + } + + } + if ( timesIterated == 0 ){ + logger.error( "Files parts that couldn't be aborted in 20 seconds are:" ); + Iterator<MultipartUpload> multipartUploadIterator = listResult.getMultipartUploads().iterator(); + while(multipartUploadIterator.hasNext()){ + logger.error( multipartUploadIterator.next().getKey() ); + } + } + } + else { + CompleteMultipartUploadRequest request = + new CompleteMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId(), + partETags ); + CompleteMultipartUploadResult amazonResult = getS3Client().completeMultipartUpload( request ); + fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); + fileMetadata.put( AssetUtils.E_TAG, amazonResult.getETag() ); + em.update( entity ); + } + } + } + + + @Override + public InputStream read( UUID appId, Entity entity, long offset, long length ) throws Exception { + + S3Object object = getS3Client().getObject( bucketName, AssetUtils.buildAssetKey( appId, entity ) ); + + byte data[] = null; + + if ( offset == 0 && length == FIVE_MB ) { + return object.getObjectContent(); + } + else { + object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length)); + } + + return new ByteArrayInputStream(data); + } + + + @Override + public InputStream read( UUID appId, Entity entity ) throws Exception { + return read( appId, entity, 0, FIVE_MB ); + } + + + @Override + public void delete( UUID appId, Entity entity ) throws Exception { + getS3Client().deleteObject(new DeleteObjectRequest(bucketName, AssetUtils.buildAssetKey( appId, entity ))); + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java deleted file mode 100644 index 5f2c041..0000000 --- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.services.assets.data; - - -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Iterator; -import java.util.Map; -import java.util.Properties; -import java.util.UUID; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.EntityManager; -import org.apache.usergrid.persistence.EntityManagerFactory; -import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider; -import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException; -import org.apache.usergrid.utils.StringUtils; - -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; - -import com.amazonaws.ClientConfiguration; -import com.amazonaws.Protocol; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.regions.Region; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; -import com.amazonaws.services.s3.model.MultipartUpload; -import com.amazonaws.services.s3.model.MultipartUploadListing; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.google.common.primitives.Ints; -import java.io.ByteArrayInputStream; -import java.io.FileInputStream; -import java.io.PushbackInputStream; -import java.util.ArrayList; -import java.util.List; -import org.apache.commons.codec.binary.Base64; - - -public class AwsSdkS3BinaryStore implements BinaryStore { - - private static final Logger logger = LoggerFactory.getLogger(AwsSdkS3BinaryStore.class ); - private static final long FIVE_MB = ( FileUtils.ONE_MB * 5 ); - - private AmazonS3 s3Client; - private String accessId; - private String secretKey; - private String bucketName; - private String regionName; - - @Autowired - private EntityManagerFactory emf; - - @Autowired - private Properties properties; - - public AwsSdkS3BinaryStore( ) { - } - - //TODO: GREY rework how the s3 client works because currently it handles initlization and returning of the client - //ideally it should only do one. and the client should be initlized at the beginning of the run. - private AmazonS3 getS3Client() throws Exception{ - - this.bucketName = properties.getProperty( "usergrid.binary.bucketname" ); - if(bucketName == null){ - logger.error( "usergrid.binary.bucketname not properly set so amazon bucket is null" ); - throw new AwsPropertiesNotFoundException( "usergrid.binary.bucketname" ); - - } - - final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider(); - AWSCredentials credentials = ugProvider.getCredentials(); - ClientConfiguration clientConfig = new ClientConfiguration(); - clientConfig.setProtocol(Protocol.HTTP); - - s3Client = new AmazonS3Client(credentials, clientConfig); - if(regionName != null) - s3Client.setRegion( Region.getRegion(Regions.fromName(regionName)) ); - - return s3Client; - } - - @Override - public void write( final UUID appId, final Entity entity, InputStream inputStream ) throws Exception { - - String uploadFileName = AssetUtils.buildAssetKey( appId, entity ); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB ); - - byte[] data = baos.toByteArray(); - - InputStream awsInputStream = new ByteArrayInputStream(data); - - final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity ); - fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() ); - - String mimeType = AssetMimeHandler.get().getMimeType( entity, data ); - - Boolean overSizeLimit = false; - - EntityManager em = emf.getEntityManager( appId ); - - - if ( written < FIVE_MB ) { // total smaller than 5mb - - ObjectMetadata om = new ObjectMetadata(); - om.setContentLength(written); - om.setContentType( mimeType ); - PutObjectResult result = null; - result = getS3Client().putObject( bucketName, uploadFileName, awsInputStream, om ); - - String md5sum = Hex.encodeHexString( Base64.decodeBase64(result.getContentMd5()) ); - String eTag = result.getETag(); - - fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); - - if(md5sum != null) - fileMetadata.put( AssetUtils.CHECKSUM, md5sum ); - fileMetadata.put( AssetUtils.E_TAG, eTag ); - - em.update( entity ); - - } - else { // bigger than 5mb... dump 5 mb tmp files and upload from them - written = 0; //reset written to 0, we still haven't wrote anything in fact - int partNumber = 1; - int firstByte = 0; - Boolean isFirstChunck = true; - List<PartETag> partETags = new ArrayList<PartETag>(); - - - //get the s3 client in order to initialize the multipart request - getS3Client(); - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest( bucketName, uploadFileName ); - InitiateMultipartUploadResult initResponse = getS3Client().initiateMultipartUpload( initRequest ); - - - InputStream firstChunck = new ByteArrayInputStream(data); - PushbackInputStream chunckableInputStream = new PushbackInputStream(inputStream, 1); - - // determine max size file allowed, default to 50mb - long maxSizeBytes = 50 * FileUtils.ONE_MB; - String maxSizeMbString = properties.getProperty( "usergrid.binary.max-size-mb", "50" ); - if ( StringUtils.isNumeric( maxSizeMbString )) { - maxSizeBytes = Long.parseLong( maxSizeMbString ) * FileUtils.ONE_MB; - } - - // always allow files up to 5mb - if (maxSizeBytes < 5 * FileUtils.ONE_MB ) { - maxSizeBytes = 5 * FileUtils.ONE_MB; - } - - while (-1 != (firstByte = chunckableInputStream.read())) { - long partSize = 0; - chunckableInputStream.unread(firstByte); - File tempFile = File.createTempFile( entity.getUuid().toString().concat("-part").concat(String.valueOf(partNumber)), "tmp" ); - - tempFile.deleteOnExit(); - OutputStream os = null; - try { - os = new BufferedOutputStream( new FileOutputStream( tempFile.getAbsolutePath() ) ); - - if(isFirstChunck == true) { - partSize = IOUtils.copyLarge( firstChunck, os, 0, ( FIVE_MB ) ); - isFirstChunck = false; - } - else { - partSize = IOUtils.copyLarge( chunckableInputStream, os, 0, ( FIVE_MB ) ); - } - written += partSize; - - if(written> maxSizeBytes){ - overSizeLimit = true; - logger.error( "OVERSIZED FILE ({}). STARTING ABORT", written ); - break; - //set flag here and break out of loop to run abort - } - } - finally { - IOUtils.closeQuietly( os ); - } - - FileInputStream chunk = new FileInputStream(tempFile); - - Boolean isLastPart = -1 == (firstByte = chunckableInputStream.read()); - if(!isLastPart) - chunckableInputStream.unread(firstByte); - - UploadPartRequest uploadRequest = new UploadPartRequest().withUploadId(initResponse.getUploadId()) - .withBucketName(bucketName) - .withKey(uploadFileName) - .withInputStream(chunk) - .withPartNumber(partNumber) - .withPartSize(partSize) - .withLastPart(isLastPart); - partETags.add( getS3Client().uploadPart(uploadRequest).getPartETag() ); - partNumber++; - } - - //check for flag here then abort. - if(overSizeLimit) { - - AbortMultipartUploadRequest abortRequest = - new AbortMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId() ); - - ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest( bucketName ); - - MultipartUploadListing listResult = getS3Client().listMultipartUploads( listRequest ); - - //upadte the entity with the error. - try { - logger.error("starting update of entity due to oversized asset"); - fileMetadata.put( "error", "Asset size is larger than max size of " + maxSizeBytes ); - em.update( entity ); - } - catch ( Exception e ) { - logger.error( "Error updating entity with error message", e ); - } - - int timesIterated = 20; - //loop and abort all the multipart uploads - while ( listResult.getMultipartUploads().size()!=0 && timesIterated > 0) { - - getS3Client().abortMultipartUpload( abortRequest ); - Thread.sleep( 1000 ); - timesIterated--; - listResult = getS3Client().listMultipartUploads( listRequest ); - if (logger.isDebugEnabled()) { - logger.debug("Files that haven't been aborted are: {}", listResult.getMultipartUploads().listIterator().toString()); - } - - } - if ( timesIterated == 0 ){ - logger.error( "Files parts that couldn't be aborted in 20 seconds are:" ); - Iterator<MultipartUpload> multipartUploadIterator = listResult.getMultipartUploads().iterator(); - while(multipartUploadIterator.hasNext()){ - logger.error( multipartUploadIterator.next().getKey() ); - } - } - } - else { - CompleteMultipartUploadRequest request = - new CompleteMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId(), - partETags ); - CompleteMultipartUploadResult amazonResult = getS3Client().completeMultipartUpload( request ); - fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); - fileMetadata.put( AssetUtils.E_TAG, amazonResult.getETag() ); - em.update( entity ); - } - } - } - - - @Override - public InputStream read( UUID appId, Entity entity, long offset, long length ) throws Exception { - - S3Object object = getS3Client().getObject( bucketName, AssetUtils.buildAssetKey( appId, entity ) ); - - byte data[] = null; - - if ( offset == 0 && length == FIVE_MB ) { - return object.getObjectContent(); - } - else { - object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length)); - } - - return new ByteArrayInputStream(data); - } - - - @Override - public InputStream read( UUID appId, Entity entity ) throws Exception { - return read( appId, entity, 0, FIVE_MB ); - } - - - @Override - public void delete( UUID appId, Entity entity ) throws Exception { - getS3Client().deleteObject(new DeleteObjectRequest(bucketName, AssetUtils.buildAssetKey( appId, entity ))); - } -}