usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [5/5] usergrid git commit: Fixed using the wrong query package. Added schedule method that can send verified objects through. First test of changed flow.
Date Mon, 26 Oct 2015 19:35:28 GMT
Fixed using the wrong query package.
Added schedule method that can send verified objects through.
First test of changed flow.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/de6d5848
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/de6d5848
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/de6d5848

Branch: refs/heads/export-feature
Commit: de6d584809785576cf59cde5a83f05113fa6e92d
Parents: 78fd382
Author: George Reyes <grey@apache.org>
Authored: Mon Oct 26 12:35:14 2015 -0700
Committer: George Reyes <grey@apache.org>
Committed: Mon Oct 26 12:35:14 2015 -0700

----------------------------------------------------------------------
 .../organizations/OrganizationResource.java     |   7 +-
 .../management/export/ExportFilter.java         |   2 +-
 .../management/export/ExportService.java        |   2 +-
 .../management/export/ExportServiceImpl.java    | 280 +++++++++++--------
 .../usergrid/management/export/export_v2.md     |   5 +
 5 files changed, 180 insertions(+), 116 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
index a8cdfa8..789cb56 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
@@ -51,7 +51,7 @@ import org.apache.usergrid.management.OrganizationInfo;
 import org.apache.usergrid.management.export.ExportFilter;
 import org.apache.usergrid.management.export.ExportFilterImpl;
 import org.apache.usergrid.management.export.ExportService;
-import org.apache.usergrid.mq.Query;
+import org.apache.usergrid.persistence.Query;
 import org.apache.usergrid.persistence.entities.Export;
 import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentials;
 import org.apache.usergrid.rest.AbstractContextResource;
@@ -369,10 +369,9 @@ public class OrganizationResource extends AbstractContextResource {
         exportFilter.setConnections( connectionSet );
         //this references core, there needs to be a better exposed way to do this
         //as well as a way to verify queries.
-        exportFilter.setQuery(Query.fromQL( query ));
-
-
+        exportFilter.setQuery( Query.fromQL( query ));
 
+        return exportFilter;
     }
 
     @GET

http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
index aff5f26..86fdf0b 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
@@ -19,7 +19,7 @@ package org.apache.usergrid.management.export;
 
 import java.util.Set;
 
-import org.apache.usergrid.mq.Query;
+import org.apache.usergrid.persistence.Query;
 
 
 /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
index db6b105..045fa7f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
@@ -37,7 +37,7 @@ public interface ExportService {
      * Schedules the export to execute using Objects that represent validated
      * credentials and filters.
      */
-    UUID schedule (Map<String,Object> json, ExportFilter exportFilter );
+    UUID schedule (Map<String,Object> json, ExportFilter exportFilter ) throws Exception;
 
 
     /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 11b5ae7..4835b63 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -311,65 +311,12 @@ public class ExportServiceImpl implements ExportService {
             return;
         }
 
-
         //extracts the filter information
         ExportFilter exportFilter = parseFilterInformation(jobData);
 
-
         //Start the beginning of the flow.
         exportApplicationsFromOrg( (UUID)config.get( "organizationId" ),config,jobExecution,s3Export,exportFilter
);
 
-    //we no longer have a concept of an application id. Just the filters from here on in.
-//        else if ( config.get( "applicationId" ) == null ) {
-//            //exports All the applications from an organization
-//            try {
-//                logger.debug( "starting export of all application from the following org
uuid: "+config.get( "organizationId" ).toString() );
-//                exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config,
jobExecution, s3Export );
-//            }
-//            catch ( Exception e ) {
-//                export.setErrorMessage( e.getMessage() );
-//                export.setState( Export.State.FAILED );
-//                em.update( export );
-//                return;
-//            }
-//        }
-//        else if ( config.get( "collectionName" ) == null ) {
-//            //exports an Application from a single organization
-//            try {
-//                logger.debug( "Starting export of application: "+ config.get( "applicationId"
).toString());
-//                exportApplicationFromOrg( ( UUID ) config.get( "organizationId" ),
-//                    ( UUID ) config.get( "applicationId" ), config, jobExecution, s3Export
);
-//            }
-//            catch ( Exception e ) {
-//                export.setErrorMessage( e.getMessage() );
-//                export.setState( Export.State.FAILED );
-//                em.update( export );
-//                return;
-//            }
-//        }
-//        else {
-//            try {
-//                //exports a single collection from an app org combo
-//                try {
-//                    logger.debug( "Starting export of the following application collection:
"+ config.get( "collectionName" ));
-//                    exportCollectionFromOrgApp( ( UUID ) config.get( "applicationId" ),
config, jobExecution,
-//                            s3Export );
-//                }
-//                catch ( Exception e ) {
-//                    export.setErrorMessage( e.getMessage() );
-//                    export.setState( Export.State.FAILED );
-//                    em.update( export );
-//                    return;
-//                }
-//            }
-//            catch ( Exception e ) {
-//                //if for any reason the backing up fails, then update the entity with a
failed state.
-//                export.setErrorMessage( e.getMessage() );
-//                export.setState( Export.State.FAILED );
-//                em.update( export );
-//                return;
-//            }
-//        }
         logger.debug( "finished the export job." );
         updateExportStatus( export,Export.State.FINISHED,null );
     }
@@ -476,45 +423,6 @@ public class ExportServiceImpl implements ExportService {
         }
     }
 
-
-    /**
-     * Exports a specific applications from an organization
-     */
-    private void exportApplicationFromOrg( UUID organizationUUID, UUID applicationId, final
Map<String, Object> config,
-                                           final JobExecution jobExecution, S3Export s3Export
) throws Exception {
-
-        //retrieves export entity
-        Export export = getExportEntity( jobExecution );
-
-        ApplicationInfo application = managementService.getApplicationInfo( applicationId
);
-        String appFileName = prepareOutputFileName( application.getName(), null );
-
-        Map ephemeral = collectionExportAndQuery( applicationId, config, export, jobExecution
);
-
-        fileTransfer( export, appFileName, ephemeral, config, s3Export );
-    }
-
-
-    /**
-     * Exports a specific collection from an org-app combo.
-     */
-    //might be confusing, but uses the /s/ inclusion or exclusion nomenclature.
-    private void exportCollectionFromOrgApp( UUID applicationUUID, final Map<String, Object>
config,
-                                             final JobExecution jobExecution, S3Export s3Export
) throws Exception {
-
-        //retrieves export entity
-        Export export = getExportEntity( jobExecution );
-        ApplicationInfo application = managementService.getApplicationInfo( applicationUUID
);
-
-        String appFileName = prepareOutputFileName( application.getName(), ( String ) config.get(
"collectionName" ) );
-
-
-        Map ephemeral = collectionExportAndQuery( applicationUUID, config, export, jobExecution
);
-
-        fileTransfer( export, appFileName, ephemeral, config, s3Export );
-    }
-
-
     /**
      * Regulates how long to wait until the next heartbeat.
      */
@@ -706,11 +614,10 @@ public class ExportServiceImpl implements ExportService {
     }
 
 
-    protected JsonGenerator getJsonGenerator( File ephermal ) throws IOException {
+    protected JsonGenerator getJsonGenerator( OutputStream entityOutputStream ) throws IOException
{
         //TODO:shouldn't the below be UTF-16?
 
-        FileOutputStream fileOutputStream = new FileOutputStream( ephermal );
-        OutputStream entityOutputStream = new BufferedOutputStream( fileOutputStream );
+        //most systems are little endian.
         JsonGenerator jg = jsonFactory.createGenerator( entityOutputStream, JsonEncoding.UTF16_LE
);
         jg.setPrettyPrinter( new MinimalPrettyPrinter( "" ) );
         jg.setCodec( new ObjectMapper() );
@@ -736,6 +643,8 @@ public class ExportServiceImpl implements ExportService {
         return outputFileName;
     }
 
+//TODO: GREY need to create a method that will be called with a filename string then generate
a output stream.
+    //That output stream will then be fed into the json generator. and added to the entitiesToExport
array.
 
     /**
      * handles the query and export of collections
@@ -750,20 +659,26 @@ public class ExportServiceImpl implements ExportService {
         int entitiesExportedCount = 0;
 
 
-        //Could easily be converted to take in input streams. Just a harder refactor.
+        //TODO:GREY Add config to change path where this file is written.
+
         List<File> entitiesToExport = new ArrayList<>(  );
-        //TODO:Add config to change path where this file is written.
         File entityFileToBeExported = new File( "tempEntityExportPart1");
-        JsonGenerator jg = getJsonGenerator( entityFileToBeExported );
         entityFileToBeExported.deleteOnExit();
+        FileOutputStream fileEntityOutputStream = new FileOutputStream( entityFileToBeExported
);
+        OutputStream entityOutputStream = new BufferedOutputStream( fileEntityOutputStream
);
         entitiesToExport.add( entityFileToBeExported );
 
+        JsonGenerator jg = getJsonGenerator( entityOutputStream );
+
+        //While this is more wordy it allows great seperation.
         List<File> connectionsToExport = new ArrayList<>(  );
         File connectionFileToBeExported = new File ("tempConnectionExportPart1");
         connectionFileToBeExported.deleteOnExit();
-        JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported
);
+        FileOutputStream fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported
);
+        OutputStream connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream
);
         connectionsToExport.add( connectionFileToBeExported );
 
+        JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionOutputStream
);
 
 
         if(exportFilter.getCollections() == null || exportFilter.getCollections().isEmpty())
{
@@ -777,12 +692,12 @@ public class ExportServiceImpl implements ExportService {
 
                     //Query entity manager for the entities in a collection
                     Query query = null;
-                    if ( config.get( "query" ) == null ) {
+                    if(exportFilter.getQuery() == null ) {
                         query = new Query();
                     }
                     else {
                         try {
-                            query = Query.fromQL( ( String ) config.get( "query" ) );
+                            query = exportFilter.getQuery();
                         }
                         catch ( Exception e ) {
                             export.setErrorMessage( e.getMessage() );
@@ -810,10 +725,11 @@ public class ExportServiceImpl implements ExportService {
                         if ( entitiesExportedCount % 1000 == 0 ) {
                             //Time to split files
                             currentFilePartIndex++;
-                            entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex
);
-                            jg = getJsonGenerator( entityFileToBeExported );
-                            entityFileToBeExported.deleteOnExit();
-                            entitiesToExport.add( entityFileToBeExported );
+                            File entityFileToBeExported2 = new File( "tempEntityExportPart"
+ currentFilePartIndex );
+                            //TODO: UNCOMMENT THIS OR THE JSON GENERATION WON'T WORK.
+                            //jg = getJsonGenerator( entityFileToBeExported2 );
+                            entityFileToBeExported2.deleteOnExit();
+                            entitiesToExport.add( entityFileToBeExported2 );
 
                             //It is quite likely that there are files that do not contain
any connections and thus there will not
 
@@ -821,7 +737,7 @@ public class ExportServiceImpl implements ExportService {
                             // about that at this point.
                             connectionFileToBeExported = new File( "tempConnectionExportPart"
+ currentFilePartIndex );
                             connectionFileToBeExported.deleteOnExit();
-                            connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported
);
+                            //connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported
);
                             connectionsToExport.add( connectionFileToBeExported );
                         }
                     }
@@ -836,12 +752,12 @@ public class ExportServiceImpl implements ExportService {
             for( String collectionName : collectionSet ) {
                 //Query entity manager for the entities in a collection
                 Query query = null;
-                if ( config.get( "query" ) == null ) {
+                if(exportFilter.getQuery() == null ) {
                     query = new Query();
                 }
                 else {
                     try {
-                        query = Query.fromQL( ( String ) config.get( "query" ) );
+                        query = exportFilter.getQuery();
                     }
                     catch ( Exception e ) {
                         export.setErrorMessage( e.getMessage() );
@@ -870,10 +786,14 @@ public class ExportServiceImpl implements ExportService {
                         //Time to split files
                         currentFilePartIndex++;
                         entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex
);
-                        jg = getJsonGenerator( entityFileToBeExported );
                         entityFileToBeExported.deleteOnExit();
+                        fileEntityOutputStream = new FileOutputStream( entityFileToBeExported
);
+                        entityOutputStream = new BufferedOutputStream( fileEntityOutputStream
);
                         entitiesToExport.add( entityFileToBeExported );
 
+                        jg = getJsonGenerator( entityOutputStream );
+
+
                         //It is quite likely that there are files that do not contain any
connections and thus there will not
 
 
@@ -881,7 +801,9 @@ public class ExportServiceImpl implements ExportService {
                         // about that at this point.
                         connectionFileToBeExported = new File( "tempConnectionExportPart"
+ currentFilePartIndex );
                         connectionFileToBeExported.deleteOnExit();
-                        connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported
);
+                        fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported
);
+                        connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream
);
+                        connectionJsonGeneration = getJsonGenerator( connectionOutputStream
);
                         connectionsToExport.add( connectionFileToBeExported );
                     }
                 }
@@ -897,6 +819,144 @@ public class ExportServiceImpl implements ExportService {
         return filePointers;
     }
 
+    //TODO: GREY need to create a method that will be called with a filename string then
generate a output stream.
+    //That output stream will then be fed into the json generator. and added to the entitiesToExport
array.
+
+    /**
+     * handles the query and export of collections
+     */
+    //TODO:Needs further refactoring.
+    protected Map collectionExportAndQuery2( UUID applicationUUID, final Map<String, Object>
config, Export export,
+                                            final JobExecution jobExecution,ExportFilter
exportFilter ) throws Exception {
+
+        EntityManager em = emf.getEntityManager( applicationUUID );
+        long starting_time = System.currentTimeMillis();
+        //The counter needs to be constant across collections since application exports do
across collection aggregation.
+        int entitiesExportedCount = 0;
+
+
+        //TODO:GREY Add config to change path where this file is written.
+
+        List<File> entitiesToExport = new ArrayList<>(  );
+        File entityFileToBeExported = new File( "tempEntityExportPart1");
+        entityFileToBeExported.deleteOnExit();
+        FileOutputStream fileEntityOutputStream = new FileOutputStream( entityFileToBeExported
);
+        OutputStream entityOutputStream = new BufferedOutputStream( fileEntityOutputStream
);
+        entitiesToExport.add( entityFileToBeExported );
+
+        JsonGenerator jg = getJsonGenerator( entityOutputStream );
+
+        //While this is more wordy it allows great seperation.
+        List<File> connectionsToExport = new ArrayList<>(  );
+        File connectionFileToBeExported = new File ("tempConnectionExportPart1");
+        connectionFileToBeExported.deleteOnExit();
+        FileOutputStream fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported
);
+        OutputStream connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream
);
+        connectionsToExport.add( connectionFileToBeExported );
+
+        JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionOutputStream
);
+
+
+        if(exportFilter.getCollections() == null || exportFilter.getCollections().isEmpty())
{
+            Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+
+            exportCollection( export, jobExecution, exportFilter, em, starting_time, entitiesExportedCount,
+                entitiesToExport, jg, connectionsToExport, connectionJsonGeneration, metadata.keySet()
);
+        }
+        //handles the case where there are specific collections that need to be exported.
+        else{
+            Set<String> collectionSet = exportFilter.getCollections();
+
+            //loop through only the collection names
+            jg = exportCollection( export, jobExecution, exportFilter, em, starting_time,
entitiesExportedCount,
+                entitiesToExport, jg, connectionsToExport, connectionJsonGeneration, collectionSet
);
+        }
+        jg.flush();
+        jg.close();
+
+        HashMap<String,List> filePointers = new HashMap<>(  );
+        filePointers.put( "entities",entitiesToExport );
+        filePointers.put( "connections",connectionsToExport );
+
+        return filePointers;
+    }
+
+
+    private JsonGenerator exportCollection( final Export export, final JobExecution jobExecution,
+                                            final ExportFilter exportFilter, final EntityManager
em, long starting_time,
+                                            int entitiesExportedCount, final List<File>
entitiesToExport,
+                                            JsonGenerator jg, final List<File> connectionsToExport,
+                                            JsonGenerator connectionJsonGeneration, final
Set<String> collectionSet )
+        throws Exception {
+        final File entityFileToBeExported;
+        final FileOutputStream fileEntityOutputStream;
+        final OutputStream entityOutputStream;
+        final File connectionFileToBeExported;
+        final FileOutputStream fileConnectionOutputStream;
+        final OutputStream connectionOutputStream;
+        for( String collectionName : collectionSet ) {
+            //Query entity manager for the entities in a collection
+            Query query = null;
+            if(exportFilter.getQuery() == null ) {
+                query = new Query();
+            }
+            else {
+                try {
+                    query = exportFilter.getQuery();
+                }
+                catch ( Exception e ) {
+                    export.setErrorMessage( e.getMessage() );
+                }
+            }
+            query.setLimit( MAX_ENTITY_FETCH );
+            query.setResultsLevel( Level.ALL_PROPERTIES );
+            query.setCollection( collectionName );
+
+            Results entities = em.searchCollection( em.getApplicationRef(), collectionName,
query );
+
+            PagingResultsIterator itr = new PagingResultsIterator( entities );
+            //counter that will inform when we should split into another file.
+            int currentFilePartIndex = 1;
+
+            for ( Object e : itr ) {
+                starting_time = checkTimeDelta( starting_time, jobExecution );
+                Entity entity = ( Entity ) e;
+                jg.writeObject( entity );
+                saveCollectionMembers( jg, em, collectionName, entity );
+                saveConnections( entity, em, connectionJsonGeneration,exportFilter );
+                jg.writeRaw( '\n' );
+                jg.flush();
+                entitiesExportedCount++;
+                if ( entitiesExportedCount % 1000 == 0 ) {
+                    //Time to split files
+                    currentFilePartIndex++;
+                    entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex
);
+                    entityFileToBeExported.deleteOnExit();
+                    fileEntityOutputStream = new FileOutputStream( entityFileToBeExported
);
+                    entityOutputStream = new BufferedOutputStream( fileEntityOutputStream
);
+                    entitiesToExport.add( entityFileToBeExported );
+
+                    jg = getJsonGenerator( entityOutputStream );
+
+
+                    //It is quite likely that there are files that do not contain any connections
and thus there will not
+
+
+                    //be anything to write to these empty connection files. Not entirely
sure what to do
+                    // about that at this point.
+                    connectionFileToBeExported = new File( "tempConnectionExportPart" + currentFilePartIndex
);
+                    connectionFileToBeExported.deleteOnExit();
+                    fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported
);
+                    connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream
);
+                    connectionJsonGeneration = getJsonGenerator( connectionOutputStream );
+                    connectionsToExport.add( connectionFileToBeExported );
+                }
+            }
+        }
+        return jg;
+    }
+
+
     private S3Export s3ExportDeterminator(final JobData jobData){
         Object s3PlaceHolder = jobData.getProperty( "s3Export" );
         S3Export s3Export = null;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
b/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
index ca86175..387d88a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
@@ -170,6 +170,11 @@ These two lines take an application and search for a specific collection
within
 
 If the query is empty/null (need to apply the fix where queries can be null, currently they
are always initialized) or if the query is "select *" then we go and search through the graph
database for each entity. If the query is more sophisticated then we query elasticsearch for
all relevant entity information.  (Maybe a useful feature would be to perform a reindexing
operation on each collection we wish to export. ) This would work exactly the same way a generic
query performed at the rest tier would access the data.
 
+###What flow does it go through?
+Well first it aggregates a bidirection map of uuid's and application names. Then checks to
see if there is any filter being applied. If not then we will export all the applications.
If there is a filter then we only check each application that is present in the filter. 
+
+Next we go to collections and check if the filter has listed any collection names to be exported.
If there aren't, then we export every single collection. If there is a filter then we export
and search for all the entities in the collection specific collections that are listed. 
+
 
 
 


Mime
View raw message