usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [01/12] git commit: Beginnings of a rebuildable index implementation and associated test.
Date Thu, 02 Oct 2014 04:12:10 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/two-dot-o 1e7da5c8e -> b2b8886ef
  refs/heads/two-dot-o-rebuildable-index a644034c7 -> 9a670e3d2


Beginnings of a rebuildable index implementation and associated test.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6a6171e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6a6171e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6a6171e8

Branch: refs/heads/two-dot-o
Commit: 6a6171e804dd7fafd85039c63401e84ca53ac8d4
Parents: 4cba3b8
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Sat Sep 27 10:04:29 2014 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Sat Sep 27 10:04:29 2014 -0400

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |   6 +-
 .../corepersistence/CpEntityManagerFactory.java |  71 +++++-----
 .../HybridEntityManagerFactory.java             |   9 +-
 .../persistence/EntityManagerFactory.java       |   9 +-
 .../cassandra/EntityManagerFactoryImpl.java     |  12 +-
 .../PerformanceEntityRebuildIndexTest.java      | 129 +++++++++++++++++++
 .../cassandra/EntityManagerFactoryImplIT.java   |   8 +-
 stack/core/src/test/resources/log4j.properties  |  13 +-
 .../usergrid/persistence/index/IndexFig.java    |   7 +
 .../persistence/index/impl/EsProvider.java      |  15 +++
 .../org/apache/usergrid/tools/IndexRebuild.java |   6 +-
 stack/tools/src/main/resources/log4j.properties |  18 ++-
 12 files changed, 246 insertions(+), 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5a5e6bc..51e660b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -211,19 +211,19 @@ public class CpEntityManager implements EntityManager {
 
     static String getCollectionScopeNameFromEntityType( String type ) {
         String csn = Schema.defaultCollectionName( type ) + COLL_SUFFIX;
-        return csn;
+        return csn.toLowerCase();
     }
 
 
     static String getCollectionScopeNameFromCollectionName( String name ) {
         String csn = name + COLL_SUFFIX;
-        return csn;
+        return csn.toLowerCase();
     }
 
 
     static String getConnectionScopeName( String entityType, String connectionType ) {
         String csn = connectionType + entityType + CONN_SUFFIX;
-        return csn;
+        return csn.toLowerCase();
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 3e9a6a9..491a47b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -48,7 +48,9 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
 import org.apache.usergrid.persistence.entities.Application;
 import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
 import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.persistence.graph.GraphManager;
 import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
 import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexScope;
@@ -69,6 +71,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeansException;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
+import rx.Observable;
 
 
 /**
@@ -581,32 +584,38 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
     }
 
 
-    public void rebuildInternalIndexes() throws Exception {
+    public void rebuildInternalIndexes( ProgressObserver po ) throws Exception {
+
+        // get all connections from systems app
+//        GraphManager gm = managerCache.getGraphManager( CpEntityManagerFactory.SYSTEM_APPS_SCOPE
);
+//
+//        Observable<String> edgeTypes = gm.getEdgeTypesFromSource( 
+//            new SimpleSearchEdgeType( systemAppId, null , null ));
 
         logger.info("Rebuilding system apps index");
         rebuildIndexScope(
                 CpEntityManagerFactory.SYSTEM_APPS_SCOPE, 
-                CpEntityManagerFactory.SYSTEM_APPS_INDEX_SCOPE );
+                CpEntityManagerFactory.SYSTEM_APPS_INDEX_SCOPE, po );
 
         logger.info("Rebuilding system orgs index");
         rebuildIndexScope(
                 CpEntityManagerFactory.SYSTEM_ORGS_SCOPE,
-                CpEntityManagerFactory.SYSTEM_ORGS_INDEX_SCOPE );
+                CpEntityManagerFactory.SYSTEM_ORGS_INDEX_SCOPE, po );
 
         logger.info("Rebuilding system props index");
         rebuildIndexScope(
                 CpEntityManagerFactory.SYSTEM_PROPS_SCOPE,
-                CpEntityManagerFactory.SYSTEM_PROPS_INDEX_SCOPE );
+                CpEntityManagerFactory.SYSTEM_PROPS_INDEX_SCOPE, po );
 
         logger.info("Rebuilding management application index");
-        rebuildApplicationIndex( MANAGEMENT_APPLICATION_ID );
+        rebuildApplicationIndex( MANAGEMENT_APPLICATION_ID, po );
 
         logger.info("Rebuilding default application index");
-        rebuildApplicationIndex( DEFAULT_APPLICATION_ID );
+        rebuildApplicationIndex( DEFAULT_APPLICATION_ID, po );
     }
 
 
-    private void rebuildIndexScope( CollectionScope cs, IndexScope is ) {
+    private void rebuildIndexScope( CollectionScope cs, IndexScope is, ProgressObserver po
) {
 
         logger.info("Rebuild index scope for {}:{}:{}", new Object[] {
             cs.getOwner(), cs.getApplication(), cs.getName()
@@ -618,38 +627,37 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
         Query q = Query.fromQL("select *");
         CandidateResults results = ei.search( q );
 
-        Map<String, UUID> appMap = new HashMap<String, UUID>();
-
         Iterator<CandidateResult> iter = results.iterator();
         while (iter.hasNext()) {
             CandidateResult cr = iter.next();
 
             Entity entity = ecm.load(cr.getId()).toBlockingObservable().last();
 
-            if (cr.getVersion().compareTo( entity.getVersion()) < 0 ) {
-                logger.warn("    Ignoring stale version uuid:{} type:{} version:{} latest
version:{}",
-                    new Object[]{
-                        cr.getId().getUuid(),
-                        cr.getId().getType(),
-                        cr.getVersion(),
-                        entity.getVersion()
+            if ( cr.getVersion().compareTo( entity.getVersion()) < 0 ) {
+                logger.warn("   Ignoring stale version uuid:{} type:{} state v:{} latest
v:{}",
+                    new Object[] { 
+                        cr.getId().getUuid(), cr.getId().getType(), 
+                        cr.getVersion(), entity.getVersion()
                     });
-                continue;
-            }
 
-            logger.info("    Updating CP Entity type: {} with id: {} for app id: {}",
-                new Object[]{cr.getId().getType(), cr.getId().getUuid(),
-                    CpEntityManagerFactory.SYSTEM_APPS_SCOPE.getApplication().getUuid()
+            } else {
+
+                logger.info("   Updating entity type {} with id {} for app {}/{}", new Object[]
{ 
+                    cr.getId().getType(), cr.getId().getUuid(), cs.getApplication().getUuid()
+                });
+
+                ei.index(entity);
+
+                if ( po != null ) {
+                    po.onProgress();
                 }
-            );
 
-            ei.index(entity);
+            }
         }
-
     }
 
 
-    public void rebuildApplicationIndex( UUID appId ) throws Exception {
+    public void rebuildApplicationIndex( UUID appId, ProgressObserver po ) throws Exception
{
 
         EntityManager em = getEntityManager( appId );
 
@@ -659,12 +667,13 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
             appId, collections.size(), collections });
 
         for ( String collection : collections ) {
-            rebuildCollectionIndex( appId, collection );
+            rebuildCollectionIndex( appId, collection, po );
         }
     }
 
 
-    public void rebuildCollectionIndex( UUID appId, String collectionName ) throws Exception
{
+    public void rebuildCollectionIndex( UUID appId, String collectionName, ProgressObserver
po ) 
+            throws Exception {
 
         logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
 
@@ -683,15 +692,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
 
             for ( org.apache.usergrid.persistence.Entity entity : r.getEntities() ) {
 
-                logger.info( "    Updating Entity name {}, type: {}, id: {} in app id: {}",
new Object[] {
+                logger.info( "   Updating Entity name {}, type: {}, id: {} in app id: {}",
new Object[] {
                         entity.getName(), entity.getType(), entity.getUuid(), appId
                 } );
 
                 try {
                     em.update( entity );
+
+                    if ( po != null ) {
+                        po.onProgress();
+                    }
                 }
                 catch ( DuplicateUniquePropertyExistsException dupee ) {
-                    logger.error( "Duplicate property for type: {} with id: {} for app id:
{}.  "
+                    logger.error( "   Duplicate property for type: {} with id: {} for app
id: {}.  "
                             + "Property name: {} , value: {}", new Object[] {
                             entity.getType(), entity.getUuid(), appId, dupee.getPropertyName(),

                             dupee.getPropertyValue()

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 3b2fd3e..8002199 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -145,12 +145,13 @@ public class HybridEntityManagerFactory implements EntityManagerFactory,
Applica
     }
 
     @Override
-    public void rebuildInternalIndexes() throws Exception {
-        factory.rebuildInternalIndexes();
+    public void rebuildInternalIndexes(ProgressObserver po) throws Exception {
+        factory.rebuildInternalIndexes(po);
     }
 
     @Override
-    public void rebuildCollectionIndex(UUID appId, String collectionName) throws Exception
{
-        factory.rebuildCollectionIndex(appId, collectionName);
+    public void rebuildCollectionIndex(UUID appId, String collectionName, ProgressObserver
po) 
+            throws Exception {
+        factory.rebuildCollectionIndex(appId, collectionName, po);
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 15e4dbe..bf4e47e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -117,12 +117,17 @@ public interface EntityManagerFactory {
 
     public void refreshIndex();
 
-    public void rebuildInternalIndexes() throws Exception;
+    public void rebuildInternalIndexes( ProgressObserver po ) throws Exception;
 
-    public void rebuildCollectionIndex( UUID appId, String collectionName ) throws Exception;
+    public void rebuildCollectionIndex( 
+        UUID appId, String collectionName, ProgressObserver po ) throws Exception;
 
     public void setApplicationContext(ApplicationContext ac);
 
     /** For testing purposes */
     public void flushEntityManagerCaches();
+
+    public interface ProgressObserver {
+        public void onProgress();
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index d1a98ec..89d8a07 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -423,12 +423,8 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory,
Applicati
     }
 
     @Override
-    public void rebuildInternalIndexes() throws Exception {
-        throw new UnsupportedOperationException("Not supported yet.");
-    }
-
-    @Override
-    public void rebuildCollectionIndex(UUID appId, String collectionName) throws Exception
{
+    public void rebuildCollectionIndex(UUID appId, String collectionName, ProgressObserver
po ) 
+            throws Exception {
 
         logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
 
@@ -468,4 +464,8 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory,
Applicati
 
     }
 
+    @Override
+    public void rebuildInternalIndexes(ProgressObserver po) throws Exception {
+        // no op
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
new file mode 100644
index 0000000..35d1d50
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence;
+
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.AbstractCoreIT;
+import org.apache.usergrid.Application;
+import org.apache.usergrid.CoreApplication;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Slf4jReporter;
+import java.util.ArrayList;
+import org.apache.usergrid.cassandra.Concurrent;
+
+
+//@RunWith(JukitoRunner.class)
+//@UseModules({ GuiceModule.class })
+@Concurrent()
+public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
+    private static final Logger logger = LoggerFactory.getLogger(PerformanceEntityRebuildIndexTest.class
);
+
+    private static final MetricRegistry registry = new MetricRegistry();
+
+    private static final long RUNTIME = TimeUnit.MINUTES.toMillis( 1 );
+
+    private static final long writeDelayMs = 7;
+    private static final long readDelayMs = 7;
+
+    @Rule
+    public Application app = new CoreApplication( setup );
+
+    private Slf4jReporter reporter;
+
+
+    @Before
+    public void startReporting() {
+
+        reporter = Slf4jReporter.forRegistry( registry ).outputTo( logger )
+                .convertRatesTo( TimeUnit.SECONDS )
+                .convertDurationsTo( TimeUnit.MILLISECONDS ).build();
+
+        reporter.start( 10, TimeUnit.SECONDS );
+    }
+
+
+    @After
+    public void printReport() {
+        reporter.report();
+        reporter.stop();
+    }
+
+
+    @Test
+    public void rebuildIndex() throws Exception {
+
+        logger.info("Started rebuildIndex()");
+
+        final EntityManager em = app.getEntityManager();
+        final long stopTime = System.currentTimeMillis() + RUNTIME;
+        final Map<String, Object> entityMap = new HashMap<>();
+
+        entityMap.put( "key1", 1000 );
+        entityMap.put( "key2", 2000 );
+        entityMap.put( "key3", "Some value" );
+
+        List<EntityRef> entityRefs = new ArrayList<EntityRef>();
+
+        int i = 0;
+        while ( System.currentTimeMillis() < stopTime ) {
+
+            entityMap.put( "key", i );
+            final Entity created = em.create("testType", entityMap );
+
+            entityRefs.add( new SimpleEntityRef( created.getType(), created.getUuid() ) );
+
+            i++;
+
+            if ( i % 1000 == 0 ) {
+                logger.debug("rebuildIndex() Created {} entities",i );
+            }
+            Thread.sleep( writeDelayMs );
+        }
+        logger.info("rebuildIndex() Created {} entities", i);
+
+        final String meterName = this.getClass().getSimpleName() + ".rebuildIndex";
+        final Meter meter = registry.meter( meterName );
+
+        EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver()
{
+            @Override
+            public void onProgress() {
+                meter.mark();
+            }
+        };
+
+        setup.getEmf().rebuildInternalIndexes( po );
+
+        setup.getEmf().rebuildCollectionIndex( app.getId(), "testTypes", po);
+
+        registry.remove( meterName );
+        logger.info("Finished rebuildIndex()");
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
index a21fea0..efd9dfb 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
@@ -26,7 +26,6 @@ import java.util.UUID;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -94,8 +93,13 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
     }
 
 
+    public void testRebuildIndexes() throws Exception {
+
+
+    }
+
+
     @Test
-    @Ignore("Fix this EntityManagerFactoryImplIT.testCreateAndGet:105->createApplication:90
ยป ApplicationAlreadyExists")
     public void testCreateAndGet() throws Exception {
         TraceTag traceTag = traceTagManager.create( "testCreateAndGet" );
         traceTagManager.attach( traceTag );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index 51f6112..2aa9983 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -39,15 +39,22 @@ log4j.logger.org.apache.usergrid.rest.security.AllowAjaxFilter=WARN, stdout
 log4j.logger.me.prettyprint.hector.api.beans.AbstractComposite=ERROR, stdout
 log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DEBUG, stdout
 
-#log4j.logger.org.apache.usergrid.persistence.PerformanceEntityReadTest=DEBUG
+log4j.logger.org.apache.usergrid.persistence.PerformanceEntityReadTest=DEBUG
+log4j.logger.org.apache.usergrid.persistence.PerformanceEntityRebuildIndexTest=DEBUG
 #log4j.logger.org.apache.usergrid.persistence=INFO
-#log4j.logger.org.apache.usergrid.corepersistence=DEBUG
+
+log4j.logger.org.apache.usergrid.corepersistence=DEBUG
+#log4j.logger.org.apache.usergrid.corepersistence.CpSetup=INFO
+#log4j.logger.org.apache.usergrid.corepersistence.CpEntityManagerFactory=DEBUG
+#log4j.logger.org.apache.usergrid.corepersistence.CpEntityManager=DEBUG
+#log4j.logger.org.apache.usergrid.corepersistence.CpRelationManager=DEBUG
+
 #log4j.logger.com.netflix.hystrix=DEBUG
 #log4j.logger.org.antlr=DEBUG
 
 #log4j.logger.org.apache.usergrid.persistence.CollectionIT=DEBUG
-#log4j.logger.org.apache.usergrid.persistence.index=DEBUG
 #log4j.logger.org.apache.usergrid.persistence.collection=DEBUG
+log4j.logger.org.apache.usergrid.persistence.index=DEBUG
 #log4j.logger.org.elasticsearch=DEBUG
 
 #log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG, stdout

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 5d59f61..142c48a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -33,6 +33,8 @@ public interface IndexFig extends GuicyFig {
 
     public static final String ELASTICSEARCH_CLUSTER_NAME = "elasticsearch.cluster_name";
 
+    public static final String ELASTICSEARCH_NODENAME = "elasticsearch.node_name";
+
     public static final String ELASTICSEARCH_INDEX_PREFIX = "elasticsearch.index_prefix";
 
     public static final String ELASTICSEARCH_STARTUP = "elasticsearch.startup";
@@ -75,4 +77,9 @@ public interface IndexFig extends GuicyFig {
     @Default( "false" ) 
     @Key( ELASTICSEARCH_FORCE_REFRESH )
     public boolean isForcedRefresh();
+
+    /** Identify the client node with a unique name. */
+    @Default("default")
+    @Key( ELASTICSEARCH_NODENAME )
+    public String getNodeName();
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
index 83a3eac..a9228ee 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
@@ -22,7 +22,10 @@ import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import java.io.File;
 import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.Properties;
+import java.util.logging.Level;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.RandomStringUtils;
 import org.apache.usergrid.persistence.core.util.AvailablePortFinder;
@@ -133,7 +136,18 @@ public class EsProvider {
 
                     // we will connect to forked ES on localhost
                     allHosts = "localhost:" + System.getProperty(LOCAL_ES_PORT_PROPNAME);
+                }
+
+                String nodeName = fig.getNodeName();
+                if ( "default".equals( nodeName )) {
+                    // no nodeName was specified, use hostname
+                    try {
+                        nodeName = InetAddress.getLocalHost().getHostName();
 
+                    } catch (UnknownHostException ex) {
+                        nodeName = "client-" + RandomStringUtils.randomAlphabetic(8);
+                        log.warn("Couldn't get hostname to use as ES node name, using " +
nodeName);
+                    }
                 }
 
                 Settings settings = ImmutableSettings.settingsBuilder()
@@ -149,6 +163,7 @@ public class EsProvider {
                     .put("client.transport.ping_timeout", 2000) // milliseconds
                     .put("client.transport.nodes_sampler_interval", 100)
                     .put("network.tcp.blocking", true)
+                    .put("node.name",  nodeName )
 
                     .build();
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java b/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
index e3d6560..81d0c1c 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
@@ -94,7 +94,7 @@ public class IndexRebuild extends ToolBase {
 
         logger.info( "Starting index rebuild" );
 
-        emf.rebuildInternalIndexes();
+        emf.rebuildInternalIndexes( null );
         emf.refreshIndex();
 
         /**
@@ -106,7 +106,7 @@ public class IndexRebuild extends ToolBase {
             Set<String> collections = getCollections( line, appId );
 
             for ( String collection : collections ) {
-                emf.rebuildCollectionIndex(appId, collection);
+                emf.rebuildCollectionIndex( appId, collection, null );
                 emf.refreshIndex();
             }
         }
@@ -135,7 +135,7 @@ public class IndexRebuild extends ToolBase {
 
         System.out.println( "Printing all apps" );
         for ( Entry<String, UUID> entry : ids.entrySet() ) {
-            System.out.println( entry.getKey() );
+            System.out.println( entry.getKey() + " appid=" + entry.getValue() );
         }
 
         return ids.values();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a6171e8/stack/tools/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/resources/log4j.properties b/stack/tools/src/main/resources/log4j.properties
index e9c23e5..b993313 100644
--- a/stack/tools/src/main/resources/log4j.properties
+++ b/stack/tools/src/main/resources/log4j.properties
@@ -18,7 +18,7 @@
 # and the pattern to %c instead of %l.  (%l is slower.)
 
 # output messages into a rolling log file as well as stdout
-log4j.rootLogger=INFO,stdout
+log4j.rootLogger=ERROR,stdout
 
 # stdout
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
@@ -26,7 +26,8 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) [%c] - %m%n
 
-log4j.category.org.apache.usergrid.tools=TRACE, stdout
+log4j.category.org.apache.usergrid.tools=TRACE
+log4j.category.org.apache.usergrid=ERROR
 
 log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN, stdout
 log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN, stdout
@@ -38,8 +39,15 @@ log4j.logger.me.prettyprint.cassandra.hector.TimingLogger=WARN, stdout
 log4j.logger.org.apache.usergrid.rest.security.AllowAjaxFilter=WARN, stdout
 log4j.logger.me.prettyprint.hector.api.beans.AbstractComposite=ERROR, stdout
 #log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DEBUG, stdout
-
-log4j.logger.org.apache.usergrid.persistence.hector.CountingMutator=INFO, stdout
-
+#log4j.logger.org.apache.usergrid.persistence.hector.CountingMutator=INFO, stdout
 #log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG, stdout
 
+#log4j.logger.org.apache.usergrid.corepersistence=INFO
+#log4j.logger.org.apache.usergrid.corepersistence.CpSetup=INFO
+log4j.logger.org.apache.usergrid.corepersistence.CpEntityManagerFactory=DEBUG
+#log4j.logger.org.apache.usergrid.corepersistence.CpEntityManager=DEBUG
+#log4j.logger.org.apache.usergrid.corepersistence.CpRelationManager=DEBUG
+
+#log4j.logger.org.apache.usergrid.persistence.collection=INFO
+#log4j.logger.org.apache.usergrid.persistence.index=DEBUG
+#log4j.logger.org.apache.usergrid.persistence.index.impl=DEBUG


Mime
View raw message