usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject usergrid git commit: Added the collection tool that needs to be expanded
Date Wed, 28 Oct 2015 18:23:38 GMT
Repository: usergrid
Updated Branches:
  refs/heads/userCollectionFix [created] a72da10f8


Added the collection tool that needs to be expanded


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a72da10f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a72da10f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a72da10f

Branch: refs/heads/userCollectionFix
Commit: a72da10f85bd01b47950e18b4ab2f47dfa4ecf23
Parents: afe0c51
Author: George Reyes <grey@apache.org>
Authored: Wed Oct 28 11:23:37 2015 -0700
Committer: George Reyes <grey@apache.org>
Committed: Wed Oct 28 11:23:37 2015 -0700

----------------------------------------------------------------------
 .../rest/applications/users/UsersResource.java  |   3 +
 ...cateUniquePropertyExistsExceptionMapper.java |   6 +
 .../usergrid/tools/CollectionUserFix.java       | 255 +++++++++++++++++++
 3 files changed, 264 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
index 6325d5a..408299a 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
@@ -100,6 +100,9 @@ public class UsersResource extends ServiceResource {
     @Path("{itemName}")
     public AbstractContextResource addNameParameter( @Context UriInfo ui, @PathParam("itemName")
PathSegment itemName )
             throws Exception {
+        //TODO: read repair if entity ref doesn't match returned entity. Detect index corruption
and delete reference if
+        //reference isn't correct.
+
 
         logger.info( "ServiceResource.addNameParameter" );
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
index b910b58..0ea1567 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
@@ -29,6 +29,12 @@ import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 public class DuplicateUniquePropertyExistsExceptionMapper
         extends AbstractExceptionMapper<DuplicateUniquePropertyExistsException> {
 
+    //when you get this exception then fire a repair task that will grab a thread from a
thread pool and async run the repair task
+    // That task will then go through and double checkk all the entities and verify that
their ref's and entities match up.
+
+
+    //npe, delete entity ref
+    //mismatched pointers, update the entity ref with whatever is in the collection. ( retrieve
via query ) .
     @Override
     public Response toResponse( DuplicateUniquePropertyExistsException e ) {
         return toResponse( BAD_REQUEST, e );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
new file mode 100644
index 0000000..829487a
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Identifier;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.entities.Application;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+
+/**
+ * Fixes issues where admin data is in organizations but can't accessed or reset the password
of. This usually returns
+ * the error "Could not find organization for email" because the user can't be found but
when the org is queried the
+ * user shows up there. If you see that then run this tool.
+ */
+public class CollectionUserFix extends ExportingToolBase {
+    private static final int PAGE_SIZE = 100;
+
+    private static final Logger logger = LoggerFactory.getLogger( CollectionUserFix.class
);
+
+    /**
+     *
+     */
+    private static final String ORGANIZATION_ARG = "org";
+
+    /**
+     *
+     */
+    private static final String APPLICATION_ARG = "app";
+
+    /**
+     *
+     */
+    private static final String QUERY_ARG = "query";
+
+    @Override
+    @SuppressWarnings( "static-access" )
+    public Options createOptions() {
+
+        Option hostOption =
+                OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription(
"Cassandra host" )
+                             .create( "host" );
+
+        Option orgOption = OptionBuilder.withArgName( ORGANIZATION_ARG ).hasArg().isRequired(
false )
+                                        .withDescription( "organization id" ).create( ORGANIZATION_ARG
);
+        Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired(
false )
+                                        .withDescription( "application uuid" ).create( APPLICATION_ARG
);
+        Option queryOption = OptionBuilder.withArgName( QUERY_ARG ).hasArg().isRequired(
false )
+                                        .withDescription( "query" ).create( QUERY_ARG );
+
+        //add a query to it
+
+        Options options = new Options();
+        options.addOption( hostOption );
+        options.addOption( orgOption );
+        options.addOption( appOption );
+        options.addOption( queryOption );
+
+        return options;
+    }
+
+
+    /**
+     * psudeo code
+     * if org id is present then ignore the application id
+     *      go through everysingle application/users entity. And check to see if it can be
queried by username.
+     * else if app id is present
+     *      go through the applications/users
+     *
+     *
+     * Take the list of applications to go through and go through them one by one. ( in the
latter else case we will only
+     *
+     * @param line
+     * @throws Exception
+     */
+
+
+    @Override
+    public void runTool( CommandLine line ) throws Exception {
+
+
+        startSpring();
+
+        logger.info( "Starting crawl of all users" );
+        System.out.println( "Starting crawl of all users" );
+        Set<Application> applicationSet = new HashSet<Application>(  );
+        EntityManager em = null;
+
+
+
+        if(line.getOptionValue( ORGANIZATION_ARG )==null||line.getOptionValue( ORGANIZATION_ARG
).isEmpty()) {
+            em = emf.getEntityManager( UUID.fromString( line.getOptionValue( APPLICATION_ARG
) ) );
+            applicationSet.add( em.getApplication() );
+        }
+        else{
+            BiMap applicationsForOrganization =
+                    managementService
+                            .getApplicationsForOrganization( UUID.fromString(line.getOptionValue(
ORGANIZATION_ARG )));
+
+            applicationSet = applicationsForOrganization.keySet();
+        }
+        startCollectionFlow( em, applicationSet, line.getOptionValue( QUERY_ARG ) );
+
+
+        logger.info( "Repair complete" );
+        System.out.println("Repair Complete");
+    }
+
+
+    //make
+    private void startCollectionFlow(final EntityManager entityManager, final Set<Application>
app, final String queryString )
+            throws Exception {// search for all orgs
+
+        Query query = new Query();
+        if(queryString!=null){
+            query = query.fromQL( queryString );
+        }
+        query.setLimit( PAGE_SIZE );
+        Results r = null;
+        EntityManager em = null;
+
+        for ( Application application : app ) {
+            //This will hold all of the applications users. This will be stored in memory
to do a simple check to see if
+            //there are any duped usernames in the collection.
+            //Memory concerns means that
+            Multimap<String, UUID> usernames = HashMultimap.create();
+
+            //This means that we need to set it for each and every single application thus
it gets set here instead of
+            //the method that calls us.
+            if(entityManager == null){
+                em = emf.getEntityManager( application.getUuid() );
+            }
+            else {
+                em = entityManager;
+            }
+
+            do {
+
+                //get all users in the management app and page for each set of a PAGE_SIZE
+                r = em.searchCollection( application, "users", query );
+                System.out.println("found "+r.size()+" number of entities");
+
+                for ( Entity entity : r.getEntities() ) {
+                    //grab all usernames returned.
+                    usernames.put( entity.getProperty( "username" ).toString().toLowerCase(),
entity.getUuid() );
+                }
+
+                query.setCursor( r.getCursor() );
+
+                logger.info( "Searching next page" );
+                System.out.println("Searching next page");
+            }
+            while ( r != null && r.size() == PAGE_SIZE );
+
+            System.out.println("Starting username crawl of "+usernames.size()+" number of
usernames");
+            //do  a get on a specific username, if it shows up more than once then remove
it
+            for ( String username : usernames.keySet() ) {
+                Collection<UUID> ids = usernames.get( username );
+
+                if ( ids.size() > 1 ) {
+                    logger.info( "Found multiple users with the username {}", username );
+                    System.out.println( "Found multiple users with the username: " + username
);
+                }
+
+                //UserInfo targetUser = managementService.getAdminUserByEmail( email );
+                Identifier identifier = new Identifier();
+                EntityRef targetUser = em.getUserByIdentifier( identifier.fromName( username
) );
+
+
+                if ( targetUser == null ) {
+                    //This means that the username isn't properly associated with targetUser
+                    List<UUID> tempIds = new ArrayList<UUID>( ids );
+                    //Collections.sort( tempIds );
+
+                    UUID toLoad = tempIds.get( 0 );
+
+                    logger.warn( "Could not load target user by username {}, loading by UUID
{} instead", username, toLoad );
+                    System.out.println( "Could not load the target user by username: " +
username
+                            + ". Loading by the following uuid instead: " + toLoad.toString()
);
+
+                    User targetUserEntity = null;
+                    try {
+                        targetUserEntity = em.get( toLoad, User.class );
+                    }catch(Exception e){
+                        System.out.println("The follow uuid has no data in this cassandra
node: "+toLoad.toString());
+                        throw e;
+                    }
+
+
+                    try {
+                        if ( targetUserEntity != null&& targetUserEntity.getUuid().equals(
toLoad )) {
+                            System.out.println("Updating uuid: "+targetUserEntity.getUuid().toString());
+                            em.update( targetUserEntity );
+                        }
+                    }
+                    catch ( DuplicateUniquePropertyExistsException dup ) {
+                        System.out.println( "Found duplicate unique property: " + dup.getPropertyName()
+ ". "
+                                + "Duplicate property is: "
+                                + dup.getPropertyValue() );
+                        //if there are duplicate unique properties then
+                        if ( dup.getPropertyName().equals( "username" ) ) {
+                            System.out.println("can I replace this with a different value
since these are duplicated in the code base");
+                            //targetUserEntity.setUsername( targetUserEntity.getUsername()
);
+                        }
+                        //else throw dup;
+                    }
+                    catch (Exception e){
+                        System.out.println("There was an issue with updating: "+e.getMessage());
+                    }
+                }
+            }
+        }
+    }
+}


Mime
View raw message