usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [27/50] [abbrv] git commit: mocked filename output. Still didn’t fix maven testFileConnection test for export. cleaned up code to only look at one org. Should make export a lot more easy to read/debug.
Date Mon, 10 Mar 2014 20:26:20 GMT
mocked filename output. Still didn’t fix maven testFileConnection test for export.
cleaned up code to only look at one org. Should make export a lot more easy to read/debug.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/152c72d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/152c72d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/152c72d4

Branch: refs/pull/70/merge
Commit: 152c72d4432f592a1fcc226ea4ac559e9dbfb56e
Parents: 88a2a48
Author: grey <greyes@apigee.com>
Authored: Thu Feb 27 17:23:03 2014 -0800
Committer: grey <greyes@apigee.com>
Committed: Thu Feb 27 17:23:03 2014 -0800

----------------------------------------------------------------------
 .../applications/ApplicationResource.java       |  2 +
 .../apache/usergrid/management/ExportInfo.java  |  4 +
 .../management/export/ExportServiceImpl.java    | 96 +++++++++++++++++---
 .../usergrid/management/export/S3Export.java    |  4 +
 .../management/export/S3ExportImpl.java         |  7 ++
 .../cassandra/ManagementServiceIT.java          | 28 ++++--
 .../management/cassandra/MockS3ExportImpl.java  | 11 ++-
 7 files changed, 130 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index 0e9d8ed..0ddf61b 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@ -228,8 +228,10 @@ public class ApplicationResource extends AbstractContextResource {
         try {
             //parse the json into some useful object (the config params)
             ExportInfo objEx = new ExportInfo( json );
+            objEx.setOrganizationId( organization.getUuid() );
             objEx.setApplicationId( applicationId );
             objEx.setCollection( colExport );
+
             jobUUID = exportService.schedule( objEx );
             uuidRet.put( "jobUUID", jobUUID.toString() );
         }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
index b1be504..d3d0097 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
@@ -22,6 +22,7 @@ public class ExportInfo extends TypedEntity {
     private String bucket_location;
     private UUID applicationId; //TODO: include application when initializing export info
     private String collection;  //TODO: include collection when initializing export info
+    private UUID organizationId;
 
 
     public ExportInfo( Map<String, Object> exportData ) {
@@ -40,6 +41,7 @@ public class ExportInfo extends TypedEntity {
         return applicationId;
     }
 
+    public UUID getOrganizationId() { return organizationId; }
 
     public String getPath() {
         return path;
@@ -74,4 +76,6 @@ public class ExportInfo extends TypedEntity {
     public void setApplicationId( UUID appId ) { applicationId = appId;}
 
     public void setCollection ( String colName) { collection = colName; }
+
+    public void setOrganizationId ( UUID orgId ) { organizationId = orgId; }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index b716293..7e5833f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -140,19 +140,15 @@ public class ExportServiceImpl implements ExportService {
         export.setState( Export.State.STARTED );
         em.update( export );
 
-        //retrieves all the organizations. Loops through them and backs up the appplications
for each one
-        Map<UUID, String> organizationGet = getOrgs( config );
-        for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
-            try {
-                //exports all the applications for a single organization
-                exportApplicationsForOrg( organization, config, jobExecution );
-            }
-            catch ( Exception e ) {
-                //if for any reason the backing up fails, then update the entity with a failed
state.
-                export.setState( Export.State.FAILED );
-                em.update( export );
-                return;
-            }
+        try {
+            //exports all the applications for a single organization
+            exportApplicationForOrg( config.getOrganizationId(), config, jobExecution );
+        }
+        catch ( Exception e ) {
+            //if for any reason the backing up fails, then update the entity with a failed
state.
+            export.setState( Export.State.FAILED );
+            em.update( export );
+            return;
         }
         export.setState( Export.State.FINISHED );
         em.update( export );
@@ -189,7 +185,6 @@ public class ExportServiceImpl implements ExportService {
         return organizationNames;
     }
 
-
     public SchedulerService getSch() {
         return sch;
     }
@@ -307,6 +302,79 @@ public class ExportServiceImpl implements ExportService {
         }
     }
 
+    //might be confusing, but uses the /s/ inclusion or exclusion nomenclature.
+    private void exportApplicationForOrg( UUID organizationUUID, final ExportInfo config,
+                                           final JobExecution jobExecution ) throws Exception
{
+
+        //retrieves export entity
+        UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
+        EntityManager exportManager = emf.getEntityManager( config.getApplicationId() );
+        Export export = exportManager.get( exportId, Export.class );
+
+        //sets up a output stream for s3 backup.
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+        // Loop through the applications per organization
+        BiMap<UUID, String> applications = managementService.getApplicationsForOrganization(
organizationUUID );
+        for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
+
+            logger.info( application.getValue() + " : " + application.getKey() );
+
+            String appFileName = prepareOutputFileName( "application", application.getValue()
);
+
+            JsonGenerator jg = getJsonGenerator( baos );
+
+            EntityManager em = emf.getEntityManager( application.getKey() );
+
+            jg.writeStartArray();
+
+            Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+            long starting_time = System.currentTimeMillis();
+
+            // Loop through the collections. This is the only way to loop
+            // through the entities in the application (former namespace).
+            //could support queries, just need to implement that in the rest endpoint.
+            for ( String collectionName : metadata.keySet() ) {
+                //if the collection you are looping through doesn't match the name of the
one you want. Don't export it.
+                if ( collectionName.equals( config.getCollection() ) ) {
+                    //Query entity manager for the entities in a collection
+                    Query query = new Query();
+                    query.setLimit( MAX_ENTITY_FETCH );
+                    query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+                    Results entities = em.searchCollection( em.getApplicationRef(), collectionName,
query );
+
+                    //pages through the query and backs up all results.
+                    PagingResultsIterator itr = new PagingResultsIterator( entities );
+                    for ( Object e : itr ) {
+                        starting_time = checkTimeDelta( starting_time, jobExecution );
+                        Entity entity = ( Entity ) e;
+                        jg.writeStartObject();
+                        jg.writeFieldName( "Metadata" );
+                        jg.writeObject(entity );
+                        saveCollectionMembers( jg, em, config.getCollection(), entity );
+                        jg.writeEndObject();
+                    }
+                }
+            }
+
+            // Close writer and file for this application.
+            jg.writeEndArray();
+            jg.close();
+            baos.flush();
+            baos.close();
+
+            //sets up the Inputstream for copying the method to s3.
+            InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+            try {
+                s3Export.copyToS3( is, config, appFileName );
+            }
+            catch ( Exception e ) {
+                export.setState( Export.State.FAILED );
+                return;
+            }
+        }
+    }
+
 
     /**
      * Regulates how long to wait until the next heartbeat.

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
index 941a612..b774db8 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
@@ -13,4 +13,8 @@ import org.apache.usergrid.management.ExportInfo;
 public interface S3Export {
     void copyToS3( InputStream inputStream, ExportInfo exportInfo, String filename );
 
+    String getFilename ();
+
+    void setFilename (String givenName);
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 2c274e3..1ca6e97 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -32,6 +32,7 @@ public class S3ExportImpl implements S3Export {
     @Override
     public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String
filename ) {
 
+
         Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
         /*won't need any of the properties as I have the export info*/
         String bucketName = exportInfo.getBucket_location();
@@ -80,4 +81,10 @@ public class S3ExportImpl implements S3Export {
             logger.error( "Error uploading to blob store", e );
         }
     }
+
+    @Override
+    public String getFilename () {return "";}
+
+    @Override
+    public void setFilename(String givenName) {;}
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index b4b82e0..70010d4 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -746,25 +746,29 @@ public class ManagementServiceIT {
 
 
     //Tests to make sure we can call the job with mock data and it runs.
-    @Ignore
+    @Ignore //Connections won't save when run with maven, but on local builds it will.
     public void testFileConnections() throws Exception {
 
         File f = null;
 
 
         try {
-            f = new File( "test.json" );
-            f.delete();
+            f = new File( "testFileConnections.json" );
+            //f.delete();
         }
         catch ( Exception e ) {
             //consumed because this checks to see if the file exists. If it doesn't then
don't do anything and carry on.
         }
 
+
         S3Export s3Export = new MockS3ExportImpl();
+        s3Export.setFilename( "testFileConnections.json" );
+
         ExportService exportService = setup.getExportService();
         HashMap<String, Object> payload = payloadBuilder();
 
         ExportInfo exportInfo = new ExportInfo( payload );
+        exportInfo.setOrganizationId( organization.getUuid() );
         exportInfo.setApplicationId( applicationId );
         exportInfo.setCollection( "users" );
 
@@ -823,7 +827,7 @@ public class ManagementServiceIT {
         File f = null;
 
         try {
-            f = new File( "test.json" );
+            f = new File( "fileValidity.json" );
             f.delete();
         }
         catch ( Exception e ) {
@@ -831,10 +835,12 @@ public class ManagementServiceIT {
         }
 
         S3Export s3Export = new MockS3ExportImpl();
+        s3Export.setFilename( "fileValidity.json" );
         ExportService exportService = setup.getExportService();
         HashMap<String, Object> payload = payloadBuilder();
 
         ExportInfo exportInfo = new ExportInfo( payload );
+        exportInfo.setOrganizationId( organization.getUuid() );
         exportInfo.setApplicationId( applicationId );
         exportInfo.setCollection( "users" );
 
@@ -870,7 +876,7 @@ public class ManagementServiceIT {
 
 
         try {
-            f = new File( "test.json" );
+            f = new File( "exportOneOrg.json" );
             f.delete();
         }
         catch ( Exception e ) {
@@ -881,10 +887,12 @@ public class ManagementServiceIT {
                      "123456789" );
 
         S3Export s3Export = new MockS3ExportImpl();
+        s3Export.setFilename("exportOneOrg.json");
         ExportService exportService = setup.getExportService();
         HashMap<String, Object> payload = payloadBuilder();
 
         ExportInfo exportInfo = new ExportInfo( payload );
+        exportInfo.setOrganizationId( organization.getUuid() );
         exportInfo.setApplicationId( applicationId );
         exportInfo.setCollection( "roles" );
 
@@ -925,7 +933,7 @@ public class ManagementServiceIT {
         String appName = "testAppNotExported";
 
         try {
-            f = new File( "test.json" );
+            f = new File( "exportOneApp.json" );
             f.delete();
         }
         catch ( Exception e ) {
@@ -949,10 +957,12 @@ public class ManagementServiceIT {
         }
 
         S3Export s3Export = new MockS3ExportImpl();
+        s3Export.setFilename( "exportOneApp.json" );
         ExportService exportService = setup.getExportService();
         HashMap<String, Object> payload = payloadBuilder();
 
         ExportInfo exportInfo = new ExportInfo( payload );
+        exportInfo.setOrganizationId( organization.getUuid() );
         exportInfo.setApplicationId( applicationId );
         exportInfo.setCollection( "roles" );
 
@@ -992,7 +1002,7 @@ public class ManagementServiceIT {
 //        String appName = "ed-app";
 
         try {
-            f = new File( "test.json" );
+            f = new File( "exportOneCollection.json" );
             f.delete();
         }
         catch ( Exception e ) {
@@ -1018,10 +1028,12 @@ public class ManagementServiceIT {
         }
 
         S3Export s3Export = new MockS3ExportImpl();
+        s3Export.setFilename( "exportOneCollection.json" );
         ExportService exportService = setup.getExportService();
         HashMap<String, Object> payload = payloadBuilder();
 
         ExportInfo exportInfo = new ExportInfo( payload );
+        exportInfo.setOrganizationId( organization.getUuid() );
         exportInfo.setApplicationId( applicationId );
         exportInfo.setCollection( "baconators" );
 
@@ -1061,6 +1073,7 @@ public class ManagementServiceIT {
         HashMap<String, Object> payload = payloadBuilder();
 
         ExportInfo exportInfo = new ExportInfo( payload );
+        exportInfo.setOrganizationId( organization.getUuid() );
         exportInfo.setApplicationId( applicationId );
 
 
@@ -1095,6 +1108,7 @@ public class ManagementServiceIT {
         JobExecution jobExecution = mock( JobExecution.class );
 
         ExportInfo exportInfo = new ExportInfo( payload );
+        exportInfo.setOrganizationId( organization.getUuid() );
         exportInfo.setApplicationId( applicationId );
 
         UUID entityExportUUID = eS.schedule( exportInfo );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index d8a8b08..6419719 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -19,6 +19,7 @@ import org.apache.usergrid.management.export.S3Export;
  * Streams / reads the information written from the export service to a file named "test.json"
  */
 public class MockS3ExportImpl implements S3Export {
+    public static String filename;
     @Override
     public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String
filename ) {
         Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
@@ -28,7 +29,7 @@ public class MockS3ExportImpl implements S3Export {
         //FileInputStream fis = new PrintWriter( inputStream );
 
         try {
-            outputStream = new FileOutputStream( new File( "test.json" ) );
+            outputStream = new FileOutputStream( new File( getFilename() ) );
         }
         catch ( FileNotFoundException e ) {
             e.printStackTrace();
@@ -44,4 +45,12 @@ public class MockS3ExportImpl implements S3Export {
             e.printStackTrace();
         }
     }
+
+    @Override
+    public String getFilename () {
+        return filename;
+    }
+
+    @Override
+    public void setFilename (String givenName) { filename = givenName; }
 }


Mime
View raw message