Merge branch 'sso-conn-pool' into USERGRID-579-jcloud171
diff --git a/stack/pom.xml b/stack/pom.xml
index 4f50779..5c3d4fe 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -95,7 +95,7 @@
<hector-version>1.1-4</hector-version>
<hector-test-version>1.1-4</hector-test-version>
<jackson-version>1.9.9</jackson-version>
- <jclouds.version>1.6.2-incubating</jclouds.version>
+ <jclouds.version>1.7.1</jclouds.version>
<jersey-version>1.18</jersey-version>
<junit-version>4.11</junit-version>
<log4j-version>1.2.16</log4j-version>
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index d5ef311..3eee96b 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -595,7 +595,7 @@
ServiceResults serviceResults = executeServiceRequest( ui, response, ServiceAction.GET, null );
Entity entity = serviceResults.getEntity();
- LOG.info( "In AssetsResource.findAsset with id: {}, range: {}, modifiedSince: {}",
+ LOG.info( "In ServiceResource.executeStreamGet with id: {}, range: {}, modifiedSince: {}",
new Object[] { entityId, rangeHeader, modifiedSince } );
Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
index b41659d..666f95e 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
@@ -139,10 +139,14 @@
UserRepo.INSTANCE.load( resource(), access_token );
byte[] data = IOUtils.toByteArray( this.getClass().getResourceAsStream( "/file-bigger-than-5M" ) );
- FormDataMultiPart form = new FormDataMultiPart().field( "file", data, MediaType.MULTIPART_FORM_DATA_TYPE );
- JsonNode node = resource().path( "/test-organization/test-app/foos" ).queryParam( "access_token", access_token )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.MULTIPART_FORM_DATA )
+ FormDataMultiPart form = new FormDataMultiPart()
+ .field( "file", data, MediaType.MULTIPART_FORM_DATA_TYPE );
+
+ JsonNode node = resource().path( "/test-organization/test-app/foos" )
+ .queryParam( "access_token", access_token )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.MULTIPART_FORM_DATA )
.post( JsonNode.class, form );
JsonNode idNode = node.get( "entities" ).get( 0 ).get( "uuid" );
@@ -151,25 +155,47 @@
logNode( node );
// get entity
- node = resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam( "access_token", access_token )
- .accept( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
+ node = resource().path( "/test-organization/test-app/foos/" + uuid )
+ .queryParam( "access_token", access_token )
+ .accept( MediaType.APPLICATION_JSON_TYPE )
+ .get( JsonNode.class );
+
logNode( node );
assertEquals( "application/octet-stream", node.findValue( AssetUtils.CONTENT_TYPE ).getTextValue() );
assertEquals( 5324800, node.findValue( AssetUtils.CONTENT_LENGTH ).getIntValue() );
idNode = node.get( "entities" ).get( 0 ).get( "uuid" );
assertEquals( uuid, idNode.getTextValue() );
- // get data
- InputStream is =
- resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam( "access_token", access_token )
- .accept( MediaType.APPLICATION_OCTET_STREAM_TYPE ).get( InputStream.class );
+ int retries = 0;
+ boolean done = false;
+ byte[] foundData = new byte[0];
- byte[] foundData = IOUtils.toByteArray( is );
+ // retry until upload complete
+ while ( !done && retries < 30 ) {
+
+ // get data
+ try {
+ InputStream is = resource().path( "/test-organization/test-app/foos/" + uuid )
+ .queryParam( "access_token", access_token )
+ .accept( MediaType.APPLICATION_OCTET_STREAM_TYPE )
+ .get( InputStream.class );
+
+ foundData = IOUtils.toByteArray( is );
+ done = true;
+
+ } catch ( Exception intentiallyIgnored ) {}
+
+ Thread.sleep(1000);
+ retries++;
+ }
+
assertEquals( 5324800, foundData.length );
// delete
- node = resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam( "access_token", access_token )
- .accept( MediaType.APPLICATION_JSON_TYPE ).delete( JsonNode.class );
+ node = resource().path( "/test-organization/test-app/foos/" + uuid )
+ .queryParam( "access_token", access_token )
+ .accept( MediaType.APPLICATION_JSON_TYPE )
+ .delete( JsonNode.class );
}
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
index e1748d3..29b5e47 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
@@ -104,7 +104,7 @@
@Override
public void write( final UUID appId, final Entity entity, InputStream inputStream ) throws IOException {
- String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
+ final String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
ByteArrayOutputStream baos = new ByteArrayOutputStream();
long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB );
byte[] data = baos.toByteArray();
@@ -112,13 +112,13 @@
final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
- String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
+ final String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
if ( written < FIVE_MB ) { // total smaller than 5mb
BlobStore blobStore = getContext().getBlobStore();
- BlobBuilder.PayloadBlobBuilder bb =
- blobStore.blobBuilder( uploadFileName ).payload( data ).calculateMD5().contentType( mimeType );
+ BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName )
+ .payload( data ).calculateMD5().contentType( mimeType );
fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
@@ -134,10 +134,11 @@
}
else { // bigger than 5mb... dump 5 mb tmp files and upload from them
- // todo: yes, AsyncBlobStore is deprecated, but there appears to be no replacement yet
- final AsyncBlobStore blobStore = getContext().getAsyncBlobStore();
+ // create temp file and copy entire file to that temp file
- File tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" );
+ LOG.debug( "Writing temp file for S3 upload" );
+
+ final File tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" );
tempFile.deleteOnExit();
OutputStream os = null;
try {
@@ -149,38 +150,52 @@
IOUtils.closeQuietly( os );
}
- BlobBuilder.PayloadBlobBuilder bb =
- blobStore.blobBuilder( uploadFileName ).payload( tempFile ).calculateMD5().contentType( mimeType );
-
fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
- if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
- bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ).toString() );
- }
- final Blob blob = bb.build();
- final File finalTempFile = tempFile;
- final ListenableFuture<String> future =
- blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
+ // JClouds no longer supports async blob store, so we have to do this fun stuff
- Runnable listener = new Runnable() {
+ LOG.debug( "Starting upload thread" );
+
+ Thread uploadThread = new Thread( new Runnable() {
@Override
public void run() {
try {
- String eTag = future.get();
+ LOG.debug( "S3 upload thread started" );
+
+ BlobStore blobStore = getContext().getBlobStore();
+
+ BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName )
+ .payload( tempFile ).calculateMD5().contentType( mimeType );
+
+ if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
+ bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ).toString() );
+ }
+ final Blob blob = bb.build();
+
+ String md5sum = Hex.encodeHexString( blob.getMetadata().getContentMetadata().getContentMD5() );
+ fileMetadata.put( AssetUtils.CHECKSUM, md5sum );
+
+ LOG.debug( "S3 upload starting" );
+
+ String eTag = blobStore.putBlob( bucketName, blob );
fileMetadata.put( AssetUtils.E_TAG, eTag );
+
+ LOG.debug( "S3 upload complete eTag=" + eTag);
+
EntityManager em = emf.getEntityManager( appId );
em.update( entity );
- finalTempFile.delete();
+ tempFile.delete();
}
catch ( Exception e ) {
LOG.error( "error uploading", e );
}
- if ( finalTempFile != null && finalTempFile.exists() ) {
- finalTempFile.delete();
+ if ( tempFile != null && tempFile.exists() ) {
+ tempFile.delete();
}
}
- };
- future.addListener( listener, executor );
+ });
+
+ uploadThread.start();
}
}
