usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [3/5] usergrid git commit: Added some logic to support new filter logic and passing it through Export.
Date Mon, 26 Oct 2015 19:35:26 GMT
Added some logic to support new filter logic and passing it through Export.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d1f71e2b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d1f71e2b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d1f71e2b

Branch: refs/heads/export-feature
Commit: d1f71e2b64c4c57f39f34451d293c4b2df1d6969
Parents: f51b73c
Author: George Reyes <grey@apache.org>
Authored: Thu Oct 22 16:44:01 2015 -0700
Committer: George Reyes <grey@apache.org>
Committed: Thu Oct 22 16:44:01 2015 -0700

----------------------------------------------------------------------
 .../organizations/OrganizationResource.java     | 32 +++++++-
 .../management/export/ExportService.java        |  6 ++
 .../management/export/ExportServiceImpl.java    | 84 +++++++++++++++-----
 3 files changed, 102 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/d1f71e2b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
index 679451a..a8cdfa8 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.rest.management.organizations;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 
 import javax.ws.rs.Consumes;
@@ -47,7 +48,10 @@ import org.apache.commons.lang.NullArgumentException;
 import org.apache.usergrid.management.ActivationState;
 import org.apache.usergrid.management.OrganizationConfig;
 import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.export.ExportFilter;
+import org.apache.usergrid.management.export.ExportFilterImpl;
 import org.apache.usergrid.management.export.ExportService;
+import org.apache.usergrid.mq.Query;
 import org.apache.usergrid.persistence.entities.Export;
 import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentials;
 import org.apache.usergrid.rest.AbstractContextResource;
@@ -298,6 +302,8 @@ public class OrganizationResource extends AbstractContextResource {
         Map<String,Object> properties;
         Map<String, Object> storage_info;
 
+        //the storage providers could be an abstract class that others can implement in order
to try to pull out
+        //their own data.
         try {
             if((properties = ( Map<String, Object> )  json.get( "properties" )) ==
null){
                 throw new NullArgumentException("Could not find 'properties'");
@@ -326,10 +332,14 @@ public class OrganizationResource extends AbstractContextResource {
                 throw new NullArgumentException( "Could not find field 's3_key'" );
             }
 
+
+
+
             //organizationid is added after the fact so that
             json.put( "organizationId",organization.getUuid());
+            ExportFilter exportFilter = exportFilterParser( json );
+            jobUUID = exportService.schedule( json,exportFilter );
 
-            jobUUID = exportService.schedule( json );
             uuidRet.put( "Export Entity", jobUUID.toString() );
         }
         catch ( NullArgumentException e ) {
@@ -345,6 +355,26 @@ public class OrganizationResource extends AbstractContextResource {
         return Response.status( SC_ACCEPTED ).entity( uuidRet ).build();
     }
 
+    //need to explore a validate query method.
+    public ExportFilter exportFilterParser(Map<String,Object> input){
+        String query = ( String ) input.get( "ql" );
+        Set applicationSet = ( Set ) input.get( "apps" );
+        Set collectionSet = ( Set ) input.get( "collections" );
+        Set connectionSet = ( Set ) input.get( "connections" );
+
+        //TODO:GREY move export filter to the top of this .
+        ExportFilter exportFilter = new ExportFilterImpl();
+        exportFilter.setApplications( applicationSet );
+        exportFilter.setCollections( collectionSet );
+        exportFilter.setConnections( connectionSet );
+        //this references core, there needs to be a better exposed way to do this
+        //as well as a way to verify queries.
+        exportFilter.setQuery(Query.fromQL( query ));
+
+
+
+    }
+
     @GET
     @RequireOrganizationAccess
     @Path("export/{exportEntity: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}")

http://git-wip-us.apache.org/repos/asf/usergrid/blob/d1f71e2b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
index 5a14012..db6b105 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
@@ -33,6 +33,12 @@ public interface ExportService {
      */
     UUID schedule( Map<String,Object> json) throws Exception;
 
+    /**
+     * Schedules the export to execute using Objects that represent validated
+     * credentials and filters.
+     */
+    UUID schedule (Map<String,Object> json, ExportFilter exportFilter );
+
 
     /**
      * Perform the export to the external resource

http://git-wip-us.apache.org/repos/asf/usergrid/blob/d1f71e2b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 2f63096..353714a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -150,6 +150,61 @@ public class ExportServiceImpl implements ExportService {
         return export.getUuid();
     }
 
+    @Override
+    public UUID schedule( final Map<String, Object> config, final ExportFilter exportFilter
) throws Exception {
+        logger.debug( "Starting to schedule the export job" );
+
+        if ( config == null ) {
+            logger.error( "export information cannot be null" );
+            return null;
+        }
+
+        EntityManager em = null;
+        try {
+            em = emf.getEntityManager( emf.getManagementAppId() );
+            Set<String> collections = em.getApplicationCollections();
+
+            if ( !collections.contains( "exports" ) ) {
+                em.createApplicationCollection( "exports" );
+            }
+        }
+        catch ( Exception e ) {
+            logger.error( "application doesn't exist within the current context" );
+            return null;
+        }
+
+        Export export = new Export();
+
+        //update state
+        try {
+            export = em.create( export );
+        }
+        catch ( Exception e ) {
+            logger.error( "Export entity creation failed" );
+            return null;
+        }
+
+        export.setState( Export.State.CREATED );
+        em.update( export );
+        //set data to be transferred to exportInfo
+        JobData jobData = new JobData();
+        jobData.setProperty( "target", config );
+        jobData.setProperty( "filters", exportFilter);
+        jobData.setProperty( EXPORT_ID, export.getUuid() );
+
+
+        long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
+
+        //schedule job
+        logger.debug( "Creating the export job with the name: "+ EXPORT_JOB_NAME );
+        sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
+
+        //update state
+        updateExportStatus( export, Export.State.SCHEDULED,null );
+
+        return export.getUuid();
+    }
+
 
     /**
      * Query Entity Manager for the string state of the Export Entity. This corresponds to
the GET /export
@@ -239,23 +294,14 @@ public class ExportServiceImpl implements ExportService {
         em.update( export );
 
         //Checks to see if the job was given a different s3 export class. ( Local or Aws
)
+        S3Export s3Export = null;
         try {
-            S3Export s3Export = s3ExportDeterminator( jobData );
+            s3Export = s3ExportDeterminator( jobData );
         }catch(Exception e) {
             updateExportStatus( export, Export.State.FAILED, e.getMessage() );
             throw e;
         }
 
-
-        //All verification of the job data should be done on the rest tier so at this point
we shouldn't need
-        //to error check.
-
-
-
-        //No longer need this specific kind of flow, but what we do need is to check the
filters
-        //the filters will tell us how we need to proceed.
-
-
         //This is defensive programming against anybody who wants to run the export job.
         //They need to add the organization id or else we won't know where the job came from
or what it has
         //access to.
@@ -267,7 +313,11 @@ public class ExportServiceImpl implements ExportService {
 
 
         //extracts the filter information
-        parseFilterInformation(jobData);
+        ExportFilter exportFilter = parseFilterInformation(jobData);
+
+
+        //Start the beginning of the flow.
+        exportApplicationsFromOrg( (UUID)config.get( "organizationId" ),config,jobExecution,s3Export,exportFilter
);
 
     //we no longer have a concept of an application id. Just the filters from here on in.
 //        else if ( config.get( "applicationId" ) == null ) {
@@ -369,7 +419,7 @@ public class ExportServiceImpl implements ExportService {
      * Exports All Applications from an Organization
      */
     private void exportApplicationsFromOrg( UUID organizationUUID, final Map<String, Object>
config,
-                                            final JobExecution jobExecution, S3Export s3Export
) throws Exception {
+                                            final JobExecution jobExecution, S3Export s3Export,
ExportFilter exportFilter ) throws Exception {
 
         //retrieves export entity
         Export export = getExportEntity( jobExecution );
@@ -774,12 +824,8 @@ public class ExportServiceImpl implements ExportService {
         }
     }
 
-    //All of this data is vaidated in the rest tier so it can be passed straight through
here
-    //TODO: GREY find a way to pass validated object data into the scheduler.
     public ExportFilter parseFilterInformation(JobData jobData){
-        Map<String,Object> filterData = ( Map<String, Object> ) jobData.getProperty(
"filters" );
-        String query = filterData.get( "ql" );
-
-
+        ExportFilter exportFilter = ( ExportFilter ) jobData.getProperty("filter");
+        return exportFilter;
     }
 }


Mime
View raw message