Added amazon s3 dependencies to get the file back out of s3 to make sure it was transferred over.
Removed smoke from smoke tests, they now properly get the file back from s3. Json validity is handled in other tests so I only verify that the object exists.
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index 5c31d43..de02659 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -553,5 +553,18 @@
<artifactId>jul-to-slf4j</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.3.3</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.7.3</version>
+ </dependency>
+
</dependencies>
</project>
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 863a8f8..c6cfc37 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -44,9 +44,12 @@
*/
public class S3ExportImpl implements S3Export {
+ String fn;
+
@Override
public void copyToS3( final InputStream inputStream, final Map<String,Object> exportInfo, String filename ) {
+ fn = filename;
Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
/*won't need any of the properties as I have the export info*/
@@ -87,13 +90,14 @@
try {
AsyncBlobStore blobStore = context.getAsyncBlobStore();
BlobBuilder blobBuilder =
- blobStore.blobBuilder( filename ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
+ blobStore.blobBuilder( fn ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
Blob blob = blobBuilder.build();
ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
+
logger.info( "Uploaded file etag=" + futureETag.get() );
}
catch ( Exception e ) {
@@ -102,8 +106,8 @@
}
@Override
- public String getFilename () {return "";}
+ public String getFilename () {return fn;}
@Override
- public void setFilename(String givenName) {;}
+ public void setFilename(String givenName) {fn = givenName;}
}
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index 6c0431b..310d46a 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -65,6 +65,13 @@
import org.apache.usergrid.utils.JsonUtils;
import org.apache.usergrid.utils.UUIDUtils;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+
import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
@@ -1473,6 +1480,9 @@
@Test //For this test please input your s3 credentials into payload builder.
public void testIntegration100EntitiesOn() throws Exception {
+ //s3client.putObject(new PutObjectRequest(bucketName, keyName, file));
+
+
S3Export s3Export = new S3ExportImpl();
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
@@ -1507,9 +1517,24 @@
when( jobExecution.getJobData() ).thenReturn( jobData );
exportService.doExport( jobExecution );
+ Thread.sleep(1000);
+
+ AWSCredentials myCredentials = new BasicAWSCredentials(
+ System.getProperty( "accessKey" ), System.getProperty("secretKey"));
+ AmazonS3 s3client = new AmazonS3Client(myCredentials);
+ S3Object sd = null;
+ try {
+ sd = s3client.getObject( new GetObjectRequest( System.getProperty( "bucketName" ),
+ s3Export.getFilename()) );
+ }catch(Exception e) {
+ assert(false);
+ }
+ assertNotNull( sd );
+
}
- @Test //For this test please input your s3 credentials into payload builder.
+ //@Ignore("I'd have to add some overhead such that I loop through all files or keep track of them all ")
+ @Test
public void testIntegration100EntitiesOnOneOrg() throws Exception {
S3Export s3Export = new S3ExportImpl();
@@ -1567,6 +1592,18 @@
when( jobExecution.getJobData() ).thenReturn( jobData );
exportService.doExport( jobExecution );
+
+ AWSCredentials myCredentials = new BasicAWSCredentials(
+ System.getProperty( "accessKey" ), System.getProperty("secretKey"));
+ AmazonS3 s3client = new AmazonS3Client(myCredentials);
+ S3Object sd = null;
+ try {
+ sd = s3client.getObject( new GetObjectRequest( System.getProperty( "bucketName" ),
+ s3Export.getFilename()) );
+ }catch(Exception e) {
+ assert(false);
+ }
+ assertNotNull( sd );
}
/*Creates fake payload for testing purposes.*/
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index 8ef1c19..a1814fd 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -42,7 +42,6 @@
int read = 0;
byte[] bytes = new byte[1024];
OutputStream outputStream = null;
- //FileInputStream fis = new PrintWriter( inputStream );
try {
outputStream = new FileOutputStream( new File( getFilename() ) );
