usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [05/18] git commit: Write, index and read performance test modules for new Core Persistence API and old Entity Manager API.
Date Mon, 24 Mar 2014 15:20:52 GMT
Write, index and read performance test modules for new Core Persistence API and old Entity Manager API.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d2cb960f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d2cb960f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d2cb960f

Branch: refs/heads/hystrix-integration
Commit: d2cb960f40de016f1c8bbaa3a54e9fbc295e7e9e
Parents: e2f6fbf
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Fri Mar 21 09:55:35 2014 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Fri Mar 21 09:55:35 2014 -0400

----------------------------------------------------------------------
 .gitignore                                      |   6 +
 stack/corepersistence/perftest1/pom.xml         |  97 ++++++
 .../usergrid/persistence/UsergridBootstrap.java | 303 +++++++++++++++++
 .../src/main/resources/log4j.properties         |  44 +++
 .../src/main/resources/project.properties       |   1 +
 .../main/resources/usergrid-test-context.xml    |  39 +++
 .../persistence/Usegrid1PerformanceIT.java      | 304 ++++++++++++++++++
 .../persistence/UsergridBootstrapTest.java      |  56 ++++
 stack/corepersistence/perftest2/pom.xml         |  85 +++++
 .../main/groovy/perftest2/CreateEntity.groovy   | 100 ++++++
 .../usergrid/persistence/CorePerformanceIT.java | 321 +++++++++++++++++++
 .../index/guice/TestIndexModule.java            |  30 ++
 12 files changed, 1386 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 38c4045..c9f4468 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,9 @@ stack/corepersistence/nbactions.xml
 stack/corepersistence/queryindex/nbactions.xml
 stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
 stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
+/stack/corepersistence/perftest2/src/main/resources/finefoods.txt
+/stack/corepersistence/perftest1/src/main/resources/finefoods.txt
+/stack/corepersistence/perftest2/nbactions.xml
+/stack/corepersistence/perftest1/nbactions.xml
+/stack/corepersistence/perftest1/src/main/resources/usergrid.properties
+/stack/corepersistence/perftest2/src/main/resources/usergrid.properties
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/pom.xml b/stack/corepersistence/perftest1/pom.xml
new file mode 100644
index 0000000..abee483
--- /dev/null
+++ b/stack/corepersistence/perftest1/pom.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" 
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.apache.usergrid</groupId>
+    <artifactId>perftest1</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.7</maven.compiler.source>
+        <maven.compiler.target>1.7</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.usergrid</groupId>
+            <artifactId>usergrid-core</artifactId>
+            <version>0.0.29-SNAPSHOT</version>
+            <type>jar</type>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.usergrid</groupId>
+            <artifactId>usergrid-test-utils</artifactId>
+            <version>0.0.29-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
+            <artifactId>groovy-all</artifactId>
+            <groupId>org.codehaus.groovy</groupId>
+            <version>2.2.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.usergrid</groupId>
+            <artifactId>usergrid-tools</artifactId>
+            <version>0.0.29-SNAPSHOT</version>
+            <scope>test</scope>
+            <type>jar</type>
+        </dependency>
+    </dependencies>
+
+    <name>perftest1</name>
+
+    <build>
+        <plugins>
+
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>2.3.2</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.codehaus.groovy</groupId>
+                        <artifactId>groovy-eclipse-compiler</artifactId>
+                        <version>2.6.0-01</version>
+                    </dependency>
+                </dependencies>
+                <configuration>
+                    <compilerId>groovy-eclipse-compiler</compilerId>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <artifactId>groovy-eclipse-compiler</artifactId>
+                <groupId>org.codehaus.groovy</groupId>
+                <version>2.6.0-01</version>
+                <extensions>true</extensions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <version>2.8</version>
+                <executions>
+                    <execution>
+                        <id>copy-dependencies</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/alternateLocation</outputDirectory>
+                            <overWriteReleases>false</overWriteReleases>
+                            <overWriteSnapshots>false</overWriteSnapshots>
+                            <overWriteIfNewer>true</overWriteIfNewer>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+        </plugins>
+    </build>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java b/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java
new file mode 100644
index 0000000..67f4af4
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.usergrid.cassandra.SchemaManager;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.StringUtils;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_APPLICATION;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_ORGANIZATION;
+
+
+/**
+ * Does the Spring magic to get bootstrap a Usergrid CassandraService, Entity Manager Factory and
+ * entity manager. Based on code from CassandraResource in the test-utils module.
+ * <p/>
+ * Note that for this resource to work properly, a project.properties file must be placed in the 
+ * src/test/resources directory with the following stanza in the project pom's build section:
+ * <p/>
+ * <testResources> <testResource> <directory>src/test/resources</directory> <filtering>true</filtering> <includes>
+ * <include>**\/*.properties</include> <include>**\/*.xml</include> </includes> </testResource> </testResources>
+ * <p/>
+ * The following property expansion macro should be placed in this project.properties file:
+ * <p/>
+ * target.directory=${pom.build.directory}
+ */
+public class UsergridBootstrap { // is that a great classname or what! 
+
+    public static final Logger LOG = LoggerFactory.getLogger( UsergridBootstrap.class );
+
+    public static final String PROPERTIES_FILE = "project.properties";
+
+    public static final String TARGET_DIRECTORY_KEY = "target.directory";
+
+    private static final Object lock = new Object();
+
+    private final File tempDir;
+    private final String schemaManagerName;
+
+    private boolean initialized = false;
+
+    private ConfigurableApplicationContext applicationContext;
+    private SchemaManager schemaManager;
+
+    private static UsergridBootstrap instance;
+    private Thread shutdown;
+
+
+    /**
+     * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the 
+     * specified SchemaManager for Cassandra.
+     */
+    UsergridBootstrap( String schemaManagerName ) throws Throwable {
+        LOG.info( "Creating CassandraResource using {} for the ClassLoader.",
+                Thread.currentThread().getContextClassLoader() );
+
+        this.schemaManagerName = schemaManagerName;
+        try {
+            this.tempDir = getTempDirectory();
+        }
+        catch ( Exception e ) {
+            LOG.error( "Failed to create temporary directory for Cassandra instance.", e );
+            throw new RuntimeException( e );
+        }
+
+        before();
+    }
+
+
+    /**
+     * Gets the temporary directory created for this CassandraResource.
+     *
+     * @return the temporary directory
+     */
+    @SuppressWarnings("UnusedDeclaration")
+    public File getTemporaryDirectory() {
+        return tempDir;
+    }
+
+
+    /**
+     * Gets a bean from the application context.
+     *
+     * @param requiredType the type of the bean
+     * @param <T> the type of the bean
+     *
+     * @return the bean
+     */
+    public <T> T getBean( String name, Class<T> requiredType ) {
+        protect();
+        return applicationContext.getBean( name, requiredType );
+    }
+
+
+    /**
+     * Gets a bean from the application context.
+     *
+     * @param requiredType the type of the bean
+     * @param <T> the type of the bean
+     *
+     * @return the bean
+     */
+    public <T> T getBean( Class<T> requiredType ) {
+        protect();
+        return applicationContext.getBean( requiredType );
+    }
+
+
+    /**
+     * Gets whether this resource is ready to use.
+     *
+     * @return true if ready to use, false otherwise
+     */
+    public boolean isReady() {
+        return initialized;
+    }
+
+    /**
+     * Protects against IDE or command line runs of a unit test which when starting the test 
+     * outside of the Suite will not start the resource. This makes sure the resource is 
+     * automatically started on a usage attempt.
+     */
+    private void protect() {
+        if ( !isReady() ) {
+            try {
+                before();
+            }
+            catch ( Throwable t ) {
+                LOG.error( "Failed to start up Cassandra.", t );
+                throw new RuntimeException( t );
+            }
+        }
+    }
+
+
+    /**
+     * before TestSuite or test Class execution.
+     *
+     * @throws Throwable if we cannot start up Cassandra
+     */
+    protected void before() throws Throwable {
+
+            String[] locations = { "usergrid-test-context.xml" };
+            applicationContext = new ClassPathXmlApplicationContext( locations );
+
+            EntityManagerFactory emf = applicationContext.getBean( EntityManagerFactory.class );
+
+            String appName = StringUtils.lowerCase( DEFAULT_APPLICATION.contains( "/" ) 
+                    ? DEFAULT_APPLICATION : DEFAULT_ORGANIZATION + "/" + DEFAULT_APPLICATION );
+
+            if ( emf.lookupApplication(appName) == null ) {
+                loadSchemaManager( schemaManagerName );
+            }
+
+            initialized = true;
+            LOG.info( "External Cassandra resource at is ready!");
+            lock.notifyAll();
+    }
+    
+
+    /** Stops Cassandra after a TestSuite or test Class executes. */
+    protected synchronized void after() {
+
+        shutdown = new Thread() {
+            @Override
+            public void run() {
+                try {
+                    Thread.currentThread().sleep( 100L );
+                }
+                catch ( InterruptedException e ) {
+                    e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
+                }
+
+                if ( schemaManager != null ) {
+                    LOG.info( "Destroying schemaManager..." );
+                    try {
+                        schemaManager.destroy();
+                    }
+                    catch ( Exception e ) {
+                        LOG.error( "Ignoring failures while dropping keyspaces: {}", e.getMessage() );
+                    }
+
+                    LOG.info( "SchemaManager destroyed..." );
+                }
+
+                applicationContext.stop();
+                LOG.info( "ApplicationContext stopped..." );
+
+            }
+        };
+
+        shutdown.start();
+    }
+
+
+    /**
+     * Loads the specified {@link SchemaManager} or the default manager if the manager 
+     * name that is provided is null.
+     *
+     * @param schemaManagerName the name of the SchemaManager to load, or null
+     */
+    private void loadSchemaManager( String schemaManagerName ) {
+        if ( !applicationContext.isActive() ) {
+            LOG.info( "Restarting context..." );
+            applicationContext.refresh();
+        }
+
+        if ( schemaManagerName != null ) {
+            LOG.info( "Looking up SchemaManager impl: {}", schemaManagerName );
+            this.schemaManager = applicationContext.getBean( schemaManagerName, SchemaManager.class );
+        }
+        else {
+            LOG.info( "The SchemaManager is not specified - using the default SchemaManager impl" );
+            this.schemaManager = applicationContext.getBean( SchemaManager.class );
+        }
+
+        schemaManager.create();
+        schemaManager.populateBaseData();
+    }
+
+
+    /**
+     * Creates a new instance
+     *
+     * @param schemaManagerName the name of the schemaManager to use
+     *
+     * @return a new CassandraResource with possibly non-default ports
+     */
+    public static UsergridBootstrap newInstance( String schemaManagerName ) throws Throwable {
+        // Uncomment to test for Surefire Failures
+        // System.setSecurityManager( new NoExitSecurityManager( System.getSecurityManager() ) );
+
+        synchronized ( lock ) {
+            if ( instance != null ) {
+                return instance;
+            }
+
+            instance = new UsergridBootstrap( schemaManagerName );
+            LOG.info("Created a new instance of CassandraResource: {}", instance);
+            return instance;
+        }
+    }
+
+
+    public static UsergridBootstrap newInstance() throws Throwable {
+        return newInstance( null );
+    }
+
+
+    /**
+     * Uses a project.properties file that Maven does substitution on to to replace the value of 
+     * a property with the path to the Maven build directory (a.k.a. target). It then uses this 
+     * path to generate a random String which it uses to append a path component to so a unique 
+     * directory is selected. If already present it's deleted, then the directory is created.
+     *
+     * @return a unique temporary directory
+     *
+     * @throws IOException if we cannot access the properties file
+     */
+    public static File getTempDirectory() throws IOException {
+        File tmpdir;
+        Properties props = new Properties();
+        props.load( ClassLoader.getSystemResourceAsStream( PROPERTIES_FILE ) );
+        File basedir = new File( ( String ) props.get( TARGET_DIRECTORY_KEY ) );
+        String comp = RandomStringUtils.randomAlphanumeric( 7 );
+        tmpdir = new File( basedir, comp );
+
+        if ( tmpdir.exists() ) {
+            LOG.info( "Deleting directory: {}", tmpdir );
+            FileUtils.forceDelete( tmpdir );
+        }
+        else {
+            LOG.info( "Creating temporary directory: {}", tmpdir );
+            FileUtils.forceMkdir( tmpdir );
+        }
+
+        return tmpdir;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/resources/log4j.properties b/stack/corepersistence/perftest1/src/main/resources/log4j.properties
new file mode 100644
index 0000000..fab8d7b
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/resources/log4j.properties
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# for production, you should probably set the root to INFO
+# and the pattern to %c instead of %l.  (%l is slower.)
+
+# output messages into a rolling log file as well as stdout
+log4j.rootLogger=INFO,stdout
+
+# stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+#log4j.appender.stdout.layout=org.apache.log4j.SimpleLayout
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) [%c] - %m%n
+
+log4j.category.org.apache=ERROR, stdout
+
+log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerFactoryImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.DaoUtils=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.ConnectionRefImpl=WARN, stdout
+log4j.logger.me.prettyprint.cassandra.hector.TimingLogger=WARN, stdout
+log4j.logger.org.apache.usergrid.rest.security.AllowAjaxFilter=WARN, stdout
+log4j.logger.me.prettyprint.hector.api.beans.AbstractComposite=ERROR, stdout
+#log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DEBUG, stdout
+
+
+#log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG, stdout
+

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/resources/project.properties b/stack/corepersistence/perftest1/src/main/resources/project.properties
new file mode 100644
index 0000000..c0bf364
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/resources/project.properties
@@ -0,0 +1 @@
+target.directory=${project.build.directory}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml b/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml
new file mode 100644
index 0000000..9bdabe6
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xmlns:util="http://www.springframework.org/schema/util"
+	xmlns:context="http://www.springframework.org/schema/context" xmlns:p="http://www.springframework.org/schema/p"
+	xmlns:hz="http://www.hazelcast.com/schema/config" xmlns:aop="http://www.springframework.org/schema/aop"
+	xsi:schemaLocation="
+	http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd
+	http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+	http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.1.xsd
+	http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd">
+	
+	<!--  configure our test properties -->
+	<bean id="properties"
+		class="org.springframework.beans.factory.config.PropertiesFactoryBean">
+		<property name="singleton" value="true" />
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="locations">
+			<list>
+				<value>classpath:/usergrid-default.properties</value>
+				<value>classpath:/usergrid.properties</value>
+				<value>classpath:/usergrid-scheduler-test.properties</value>
+			</list>
+		</property>
+	</bean>
+
+	<import resource="classpath:/usergrid-core-context.xml"/>
+
+  <bean id="setup" class="org.apache.usergrid.persistence.cassandra.Setup">
+    <constructor-arg ref="entityManagerFactory"/>
+    <constructor-arg ref="cassandraService"/>
+  </bean>
+
+  <!-- Refer to a named schemaManager from the DataControl annotation thusly -->
+  <bean id="coreManager" class="org.apache.usergrid.persistence.CoreSchemaManager">
+    <constructor-arg ref="setup"/>
+    <constructor-arg ref="cassandraCluster"/>
+  </bean>
+</beans>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
new file mode 100644
index 0000000..af47a8c
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.lang.NumberUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class Usegrid1PerformanceIT {
+    private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceIT.class);
+
+    // max entities we will write and read
+    static int maxEntities = Integer.MAX_VALUE;
+
+    // each app will get all data
+    static int orgCount = 2;
+    static int appCount = 5  ;
+
+    // number of threads = orgCount x appCount 
+
+    // total number of records = orgCount x appCount x numRecords
+
+    private final EntityManagerFactory emf;
+
+
+    public Usegrid1PerformanceIT() throws Throwable {
+        emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
+    }
+   
+
+    public EntityManagerFactory getEmf() {
+        return emf;
+    }
+
+
+    @Ignore
+    @Test
+    public void loadAndReadData() throws Exception {
+
+        log.info("Start Data Load");
+        List<UUID> apps = loadData();
+        log.info("Finish Data Load");
+
+        log.info("Start Data Read");
+        readData( apps );
+        log.info("Finish Data Read");
+
+        runSelectedQueries( apps );
+
+    }
+
+
+    private List<UUID> loadData() throws Exception {
+
+        long time = new Date().getTime();
+
+        List<UUID> apps = new ArrayList<UUID>();
+        List<Thread> threads = new ArrayList<Thread>();
+
+        for ( int i=0; i<orgCount; i++ ) {
+
+            for ( int j=0; j<appCount; j++ ) {
+
+                UUID appId = getEmf().createApplication(
+                    "testorg-" + RandomStringUtils.randomAlphanumeric(6), 
+                    "testapp-" + RandomStringUtils.randomAlphanumeric(6));
+
+                apps.add( appId );
+
+                Thread t = new Thread( new DataLoader( appId ));
+                t.start();
+                threads.add(t);
+            }
+        }
+
+        // wait for indexing to end
+        for ( Thread t : threads ) {
+            t.join();
+        }
+
+        return apps;
+    }
+
+
+    private void readData( List<UUID> apps ) throws InterruptedException {
+
+        List<Thread> threads = new ArrayList<Thread>();
+        for ( UUID app : apps ) {
+
+            Thread t = new Thread( new DataReader( app ));
+            t.start();
+            threads.add(t);
+        }
+
+        // wait for reading to end
+        for ( Thread t : threads ) {
+            t.join();
+        }
+    }
+
+
+    private class DataReader implements Runnable {
+        UUID app;
+
+        public DataReader( UUID app ) {
+            this.app = app;
+        }
+
+        public void run() {
+
+            final EntityManager em;
+            try {
+                em = getEmf().getEntityManager( app );
+            } catch (Throwable ex) {
+                log.error("Error getting Entity Manager, aborting", ex);
+                return;
+            }
+
+            UUID appId = app;
+
+            Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+            query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+            Results results;
+            try {
+                results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+            } catch (Exception ex) {
+                log.error("Error on search, aborting", ex);
+                return;
+            }
+
+            results.getEntities(); // cause retrieval from Cassandra
+            int count = results.size();
+
+            while ( results.hasCursor() && count < maxEntities ) {
+                query.setCursor( results.getCursor() )   ;
+                try {
+                    results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+                } catch (Exception ex) {
+                    log.error("Error on search, aborting", ex);
+                    log.info("Read {} reviews in {}", count, appId );
+                    return;
+                }
+                results.getEntities(); // cause retrieval from Cassanda;
+                count += results.size();
+
+                log.info("Read {} reviews in {}", count, appId );
+            }
+        }
+    }
+
+
+    private class DataLoader implements Runnable {
+        UUID app;
+
+        public DataLoader( UUID scope ) {
+            this.app = scope;
+        }
+
+        public void run() {
+
+            final EntityManager em;
+            try {
+                em = getEmf().getEntityManager( app );
+            } catch (Throwable ex) {
+                log.error("Error getting Entity Manager, aborting", ex);
+                return;
+            }
+
+            BufferedReader br;
+            try {
+                InputStreamReader isr = new InputStreamReader( 
+                    getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+                br = new BufferedReader(isr);
+            } catch (Exception ex) {
+                throw new RuntimeException("Error opening file", ex);
+            }
+            String s = null;
+
+            // create the first entry
+            Map<String, Object> currentEntityMap = new HashMap<String, Object>();
+
+            UUID appId = app;
+
+            int count = 0;
+            try {
+                while ( (s = br.readLine()) != null && count < maxEntities ) {
+                    
+                    try {
+                        
+                        if ( s.trim().equals("")) { // then we are at end of a record
+                            
+                            // write and index current entity
+                            Entity entity = em.create("review", currentEntityMap );
+                            
+                            if ( maxEntities < 20 ) {
+                                log.info("Index written for {}", entity.getUuid());
+                                log.info("---");
+                            }
+                            
+                            // create the next entity
+                            currentEntityMap = new HashMap<String, Object>();
+                            
+                            count++;
+                            if (count % 100000 == 0) {
+                                log.info("Indexed {} reviews in {}", count, appId );
+                            }
+                            continue;
+                        }
+                        
+                        // process a field
+                        String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+                        String value = s.substring( s.indexOf(":") + 1 ).trim();
+                        
+                        if ( maxEntities < 20 ) {
+                            log.info("Indexing {} = {}", name, value);
+                        }
+                        
+                        if ( NumberUtils.isNumber(value) && value.contains(".")) {
+                            currentEntityMap.put( name, Double.parseDouble(value));
+                            
+                        } else if ( NumberUtils.isNumber(value) ) {
+                            currentEntityMap.put( name, Long.parseLong(value));
+                            
+                        } else {
+                            currentEntityMap.put( name, value.toString() );
+                        } 
+
+                    } catch ( Exception e ) {
+                        log.info("Error on line " + count);
+                    }
+                }
+
+            } catch (IOException ex) {
+                throw new RuntimeException("Error reading file", ex);
+            }
+        }
+    }   
+
+
+    public void runSelectedQueries( List<UUID> apps ) throws Exception { 
+
+        for ( UUID app : apps ) {
+
+            final EntityManager em;
+            try {
+                em = getEmf().getEntityManager( app );
+            } catch (Throwable ex) {
+                log.error("Error getting Entity Manager, aborting", ex);
+                return;
+            }
+
+            query(em, "product_productid = 'B006K2ZZ7K'") ;
+            query(em, "review_profilename = 'Twoapennything'") ;
+            query(em, "review_profilename contains 'Natalia'") ;
+            query(em, "review_profilename contains 'Patrick'") ;
+            query(em, "review_time = 1342051200") ;
+            query(em, "review_time > 1342051200") ;
+            query(em, "review_score > 0");
+            query(em, "review_score > 2");
+            query(em, "review_score > 3");
+            query(em, "review_score > 4");
+            query(em, "review_score > 5");
+        }
+    }
+
+    public static void query( EntityManager em, String query ) throws Exception {
+        Query q = Query.fromQL(query) ;
+        Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
+        log.info("size = {} returned from query {}",results.size(), q.getQl() );
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java
new file mode 100644
index 0000000..dfdac5d
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java
@@ -0,0 +1,56 @@
+
+package org.apache.usergrid.persistence;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class UsergridBootstrapTest {
+    private static final Logger log = LoggerFactory.getLogger(UsergridBootstrapTest.class);
+   
+    @Ignore // just an example and not included in performance test 
+    @Test
+    public void testBasicOperation() throws Throwable {
+
+        UsergridBootstrap sbf = UsergridBootstrap.newInstance();
+        EntityManagerFactory emf = sbf.getBean( EntityManagerFactory.class );
+
+        UUID appId = emf.createApplication(
+            "testorg-" + RandomStringUtils.randomAlphanumeric(6), 
+            "testapp-" + RandomStringUtils.randomAlphanumeric(6));
+
+        EntityManager em = emf.getEntityManager( appId );
+
+        Map<String, Object> map = new HashMap<String, Object>() {{
+            put("name", RandomStringUtils.randomAlphanumeric(6));
+            put("timestamp", new Date().getTime());
+        }};
+        Entity entity = em.create( "testentity", map );
+        String name = (String)entity.getProperty( "name" );
+
+        Entity got = em.get( entity.getUuid() );
+        assertNotNull(got);
+        String returnedName = (String)got.getProperty("name");
+        long timestamp = (long)got.getProperty("timestamp");
+        assertEquals( name, returnedName );
+        log.info(">>>> Got back name={} : time={}",  returnedName, timestamp);
+
+        Query query = Query.fromQL("name = '" + name + "'");
+        Entity found = em.searchCollection( em.getApplication(), "testentities", query ).getEntity();
+        assertNotNull(found);
+        returnedName = (String)found.getProperty("name");
+        timestamp = (long)found.getProperty("timestamp");
+        assertEquals( name, returnedName );
+        log.info(">>>> Found name={} : time={}",  returnedName, timestamp);
+    }
+    
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/pom.xml b/stack/corepersistence/perftest2/pom.xml
new file mode 100644
index 0000000..be1a457
--- /dev/null
+++ b/stack/corepersistence/perftest2/pom.xml
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" 
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.apache.usergrid</groupId>
+    <artifactId>perftest2</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.7</maven.compiler.source>
+        <maven.compiler.target>1.7</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.usergrid</groupId>
+            <artifactId>queryindex</artifactId>
+            <version>1.0-SNAPSHOT</version>
+            <type>jar</type>
+        </dependency>
+
+        <dependency>
+            <artifactId>groovy-all</artifactId>
+            <groupId>org.codehaus.groovy</groupId>
+            <version>2.2.2</version>
+        </dependency>
+
+    </dependencies>
+
+    <name>perftest2</name>
+
+    <build>
+        <plugins>
+
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>2.3.2</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.codehaus.groovy</groupId>
+                        <artifactId>groovy-eclipse-compiler</artifactId>
+                        <version>2.6.0-01</version>
+                    </dependency>
+                </dependencies>
+                <configuration>
+                    <compilerId>groovy-eclipse-compiler</compilerId>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <artifactId>groovy-eclipse-compiler</artifactId>
+                <groupId>org.codehaus.groovy</groupId>
+                <version>2.6.0-01</version>
+                <extensions>true</extensions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <version>2.8</version>
+                <executions>
+                    <execution>
+                        <id>copy-dependencies</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/alternateLocation</outputDirectory>
+                            <overWriteReleases>false</overWriteReleases>
+                            <overWriteSnapshots>false</overWriteSnapshots>
+                            <overWriteIfNewer>true</overWriteIfNewer>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+        </plugins>
+    </build>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy b/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy
new file mode 100644
index 0000000..d1a6477
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package perftest2;
+
+import com.google.inject.Injector
+import com.netflix.config.ConfigurationManager
+import org.apache.commons.lang3.RandomStringUtils
+import org.apache.usergrid.persistence.collection.CollectionScope
+import org.apache.usergrid.persistence.collection.EntityCollectionManager
+import com.google.inject.Guice
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl
+import org.apache.usergrid.persistence.collection.migration.MigrationManager
+import org.apache.usergrid.persistence.collection.util.EntityUtils
+import org.apache.usergrid.persistence.index.EntityCollectionIndex
+import org.apache.usergrid.persistence.index.guice.IndexModule
+import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory
+import org.apache.usergrid.persistence.model.entity.Entity
+import org.apache.usergrid.persistence.model.entity.Id
+import org.apache.usergrid.persistence.model.entity.SimpleId
+import org.apache.usergrid.persistence.model.field.LongField
+import org.apache.usergrid.persistence.model.field.StringField
+import org.apache.usergrid.persistence.model.util.UUIDGenerator
+import org.apache.usergrid.persistence.query.Query
+import org.apache.usergrid.persistence.query.Results
+
+
+/** 
+ * Just an example and not included in performance test.
+ */
+public class CreateEntity {
+
+    public static void main( String[] args ) {
+
+        // setup core persistence
+        ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" )
+        Injector injector = Guice.createInjector( new IndexModule() )
+
+        // only on first run
+        //MigrationManager m = injector.getInstance( MigrationManager.class )
+        //m.migrate()
+
+        EntityCollectionManagerFactory ecmf =
+            injector.getInstance( EntityCollectionManagerFactory.class )
+        EntityCollectionIndexFactory ecif =
+            injector.getInstance( EntityCollectionIndexFactory.class )
+
+        // create scope
+        Id appId = new SimpleId("entitytest")
+        Id orgId = new SimpleId("usergrid")
+        CollectionScope scope = new CollectionScopeImpl( appId, orgId, "testentities" )
+
+        // entity manager and index interfaces
+        EntityCollectionManager ecm = ecmf.createCollectionManager( scope )
+        EntityCollectionIndex eci = ecif.createCollectionIndex( scope )
+
+        // create entity
+        Entity entity = new Entity(new SimpleId(UUIDGenerator.newTimeUUID(), "testentity"))
+        String name = RandomStringUtils.randomAlphanumeric(10) 
+        entity.setField( new StringField("name", name))
+        entity.setField( new LongField("timestamp", new Date().getTime()))
+
+        // write and index entity
+        ecm.write( entity ).toBlockingObservable().last()
+        eci.index( entity )
+
+        eci.refresh()
+
+        ecm = ecmf.createCollectionManager( scope )
+        eci = ecif.createCollectionIndex( scope )
+
+        // get back entity
+        def got = ecm.load( entity.getId() ).toBlockingObservable().last()
+
+        def returnedName = got.getField("name").getValue()
+        def timestamp = got.getField("timestamp").getValue()
+        println ">>>> Got back name=${returnedName} : time=${timestamp}"
+
+        // search for entity
+        def found = eci.execute( Query.fromQL( "name = '${name}'".toString() ) ).getEntities().get(0)
+
+        returnedName = found.getField("name").getValue()
+        timestamp = found.getField("timestamp").getValue()
+        println ">>>> Found name=${returnedName} : time=${timestamp}"
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
new file mode 100644
index 0000000..8536fca
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.netflix.config.ConfigurationManager;
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.index.EntityCollectionIndex;
+import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.query.Query;
+import org.apache.usergrid.persistence.query.Results;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class CorePerformanceIT {
+    private static final Logger log = LoggerFactory.getLogger(CorePerformanceIT.class);
+
+    // max entities we will write and read
+    static int maxEntities = Integer.MAX_VALUE;
+
+    // each app will get all data
+    static int orgCount = 2;
+    static int appCount = 5  ;
+
+    // number of threads = orgCount x appCount 
+
+    // total number of records = orgCount x appCount x numRecords
+
+    private final EntityCollectionManagerFactory ecmf;
+    private final EntityCollectionIndexFactory ecif;
+
+    public CorePerformanceIT() {
+        Injector injector = Guice.createInjector( new TestIndexModule() );
+        ecmf = injector.getInstance( EntityCollectionManagerFactory.class );
+        ecif = injector.getInstance( EntityCollectionIndexFactory.class );
+    }
+
+    @Ignore
+    @Test
+    public void loadAndReadData() throws IOException, InterruptedException {
+
+        ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" );
+
+        // only on first run
+        //MigrationManager m = injector.getInstance( MigrationManager.class )
+        //m.migrate()
+
+        log.info("Start Data Load");
+        List<CollectionScope> scopes = loadData();
+        log.info("Finish Data Load");
+
+        log.info("Start Data Read");
+        readData( scopes );
+        log.info("Finish Data Read");
+
+        runSelectedQueries( scopes );
+
+    }
+
+
+    private List<CollectionScope> loadData() throws InterruptedException {
+
+        long time = new Date().getTime();
+
+        List<CollectionScope> scopes = new ArrayList<CollectionScope>();
+        List<Thread> threads = new ArrayList<Thread>();
+
+        for ( int i=0; i<orgCount; i++ ) {
+
+            String orgName = "org-" + i + "-" + time;
+            final Id orgId = new SimpleId(orgName);
+
+            for ( int j=0; j<appCount; j++ ) {
+
+                String appName = "app-" + j + "-" + time;
+                final Id appId = new SimpleId(appName);
+
+                CollectionScope scope = new CollectionScopeImpl( orgId, appId, "reviews" );
+                scopes.add( scope );
+
+                Thread t = new Thread( new DataLoader( scope ));
+                t.start();
+                threads.add(t);
+            }
+        }
+
+        // wait for indexing to end
+        for ( Thread t : threads ) {
+            t.join();
+        }
+
+        return scopes;
+    }
+
+
+    private void readData( List<CollectionScope> scopes ) throws InterruptedException {
+
+        List<Thread> threads = new ArrayList<Thread>();
+        for ( CollectionScope scope : scopes ) {
+
+            Thread t = new Thread( new DataReader( scope ));
+            t.start();
+            threads.add(t);
+        }
+
+        // wait for reading to end
+        for ( Thread t : threads ) {
+            t.join();
+        }
+    }
+
+    /**
+     * @return the ecmf
+     */
+    public EntityCollectionManagerFactory getEcmf() {
+        return ecmf;
+    }
+
+    /**
+     * @return the ecif
+     */
+    public EntityCollectionIndexFactory getEcif() {
+        return ecif;
+    }
+
+
+    private class DataReader implements Runnable {
+        CollectionScope scope;
+
+        public DataReader( CollectionScope scope ) {
+            this.scope = scope;
+        }
+
+        public void run() {
+
+            Id orgId = scope.getOrganization();
+            Id appId = scope.getOwner();
+
+            EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+            EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+            Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+            query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+            Results results = eci.execute( query );
+            results.getEntities(); // cause retrieval from Cassandra
+            int count = results.size();
+
+            while ( results.hasCursor() && count < maxEntities ) {
+                query.setCursor( results.getCursor() )   ;
+                results = eci.execute( query );
+                results.getEntities(); // cause retrieval from Cassanda;
+                count += results.size();
+
+                log.info("Read {} reviews in {} / {} ", count, orgId, appId );
+            }
+        }
+    }
+
+
+    private class DataLoader implements Runnable {
+        CollectionScope scope;
+
+        public DataLoader( CollectionScope scope ) {
+            this.scope = scope;
+        }
+
+        public void run() {
+
+            EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+            EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+            BufferedReader br;
+            try {
+                InputStreamReader isr = new InputStreamReader( 
+                    getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+                br = new BufferedReader(isr);
+            } catch (Exception ex) {
+                throw new RuntimeException("Error opening file", ex);
+            }
+            String s = null;
+
+            // create the first entry
+            Entity current = new Entity(
+                new SimpleId(UUIDGenerator.newTimeUUID(), "review")); 
+
+            Id orgId = scope.getOrganization();
+            Id appId = scope.getOwner();
+
+            int count = 0;
+            try {
+                while ( (s = br.readLine()) != null && count < maxEntities ) {
+                    
+                    try {
+                        
+                        if ( s.trim().equals("")) { // then we are at end of a record
+                            
+                            // write and index current entity
+                            ecm.write( current ).toBlockingObservable().last();
+                            eci.index( current );
+                            
+                            if ( maxEntities < 20 ) {
+                                log.info("Index written for {}", current.getId());
+                                log.info("---");
+                            }
+                            
+                            // create the next entity
+                            current = new Entity(
+                                    new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+                            
+                            count++;
+                            if (count % 100000 == 0) {
+                                log.info("Indexed {} reviews in {} / {} ", count, orgId, appId );
+                            }
+                            continue;
+                        }
+                        
+                        // process a field
+                        String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+                        String value = s.substring( s.indexOf(":") + 1 ).trim();
+                        
+                        if ( maxEntities < 20 ) {
+                            log.info("Indexing {} = {}", name, value);
+                        }
+                        
+                        if ( NumberUtils.isNumber(value) && value.contains(".")) {
+                            current.setField( new DoubleField( name, Double.parseDouble(value)));
+                            
+                        } else if ( NumberUtils.isNumber(value) ) {
+                            current.setField( new LongField( name, Long.parseLong(value)));
+                            
+                        } else {
+                            current.setField( new StringField( name, value.toString() ));
+                        }
+
+                    } catch ( Exception e ) {
+                        log.info("Error on line " + count);
+                    }
+                }
+
+            } catch (IOException ex) {
+                throw new RuntimeException("Error reading file", ex);
+            }
+
+            eci.refresh();
+        }
+    }   
+
+
+    public void runSelectedQueries( List<CollectionScope> scopes ) { 
+
+        for ( CollectionScope scope : scopes ) {
+
+            EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+            EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+            // TODO: come up with more and more complex queries for CorePerformanceIT
+
+            query(eci, "product_productid = 'B006K2ZZ7K'") ;
+            query(eci, "review_profilename = 'Twoapennything'") ;
+            query(eci, "review_profilename contains 'Natalia'") ;
+            query(eci, "review_profilename contains 'Patrick'") ;
+            query(eci, "review_time = 1342051200") ;
+            query(eci, "review_time > 1342051200") ;
+            query(eci, "review_score > 0");
+            query(eci, "review_score > 2");
+            query(eci, "review_score > 3");
+            query(eci, "review_score > 4");
+            query(eci, "review_score > 5");
+        }
+    }
+
+    public static void query( EntityCollectionIndex eci, String query ) {;
+        Query q = Query.fromQL(query) ;
+        Results results = eci.execute( q );
+        log.info("size = {} returned from query {}",results.size(), q.getQl() );
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
new file mode 100644
index 0000000..3c2390b
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.index.guice;
+
+import org.apache.usergrid.persistence.collection.guice.TestModule;
+
+
+public class TestIndexModule extends TestModule {
+
+    @Override
+    protected void configure() {
+        install( new IndexModule() );
+    }
+}


Mime
View raw message