gora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From draz...@apache.org
Subject gora git commit: GORA-424 Cache cursor size to improve performance.
Date Wed, 10 Jun 2015 20:36:30 GMT
Repository: gora
Updated Branches:
  refs/heads/master 903aeb016 -> bb09d8916


GORA-424 Cache cursor size to improve performance.

- Cache cursor size to improve performance in MongoStore
- Avoid integer division bug
- Added result progress test

This closes apache/gora#26 PR.

Squashed commit of the following:

commit 8d47c13096d15f1936d08b69757b1973ea781955
Author: Alexander Yastrebov <yastrebov.alex@gmail.com>
Date:   Wed Jun 10 11:37:35 2015 +0300

    Fixed integer division bug. Added result progress test

commit 1ebf813a1bbb803aff6bbcd15bbd7e6ee83b788b
Author: Alexander Yastrebov <yastrebov.alex@gmail.com>
Date:   Tue Jun 9 19:51:57 2015 +0300

    Cache cursor size


Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/bb09d891
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/bb09d891
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/bb09d891

Branch: refs/heads/master
Commit: bb09d8916736e3e572eef2cb5e79fe11c80ad1de
Parents: 903aeb0
Author: Damien Raude-Morvan <drazzib@drazzib.com>
Authored: Wed Jun 10 22:33:14 2015 +0200
Committer: Damien Raude-Morvan <drazzib@drazzib.com>
Committed: Wed Jun 10 22:33:14 2015 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../gora/mongodb/query/MongoDBResult.java       | 20 ++---
 .../gora/mongodb/store/TestMongoStore.java      | 78 ++++++++++++++++++--
 3 files changed, 84 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/gora/blob/bb09d891/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 12d036e..1ac61d5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -5,6 +5,8 @@
 Apache Gora 0.6.1 Release - 02/03/2015 (dd/mm/yyyy)
 Release Report - http://s.apache.org/l69 
 
+* GORA-424 Cache cursor size to improve performance (Alexander Yastrebov via drazzib)
+
 * GORA-423 BSONDecorator returns empty string for null field value (Alexander Yastrebov via
drazzib)
 
 * GORA-262 Add support for HTTPClient authentication in gora-solr (Furkan KAMACI via lewismc)

http://git-wip-us.apache.org/repos/asf/gora/blob/bb09d891/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java
----------------------------------------------------------------------
diff --git a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java
index 5524345..a032ec0 100644
--- a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java
+++ b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java
@@ -31,17 +31,18 @@ import com.mongodb.DBObject;
 /**
  * MongoDB specific implementation of the {@link org.apache.gora.query.Result}
  * interface.
- * 
+ *
  * @author Fabien Poulard <fpoulard@dictanova.com>
  * @author Damien Raude-Morvan <draudemorvan@dictanova.com>
  */
 public class MongoDBResult<K, T extends PersistentBase> extends
-    ResultBase<K, T> {
+        ResultBase<K, T> {
 
   /**
    * Reference to the cursor pointing to the results
    */
   private DBCursor cursor;
+  private int size;
 
   public MongoDBResult(DataStore<K, T> dataStore, Query<K, T> query) {
     super(dataStore, query);
@@ -49,12 +50,13 @@ public class MongoDBResult<K, T extends PersistentBase> extends
 
   @Override
   public float getProgress() throws IOException {
-    if (cursor == null)
+    if (cursor == null) {
       return 0;
-    else if (cursor.size() == 0)
+    } else if (size == 0) {
       return 1;
-    else
-      return offset / cursor.size();
+    } else {
+      return offset / (float) size;
+    }
   }
 
   @Override
@@ -73,19 +75,19 @@ public class MongoDBResult<K, T extends PersistentBase> extends
     DBObject obj = cursor.next();
     key = (K) obj.get("_id");
     persistent = ((MongoStore<K, T>) getDataStore()).newInstance(obj,
-        getQuery().getFields());
+            getQuery().getFields());
     return persistent != null;
   }
 
   /**
    * Save the reference to the cursor that holds the actual results.
-   * 
+   *
    * @param cursor
    *          {@link DBCursor} obtained from a query execution and that holds
    *          the actual results
    */
   public void setCursor(DBCursor cursor) {
     this.cursor = cursor;
+    this.size = cursor.size();
   }
-
 }

http://git-wip-us.apache.org/repos/asf/gora/blob/bb09d891/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java
----------------------------------------------------------------------
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java
b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java
index e29d4db..f2cc6ce 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java
@@ -17,7 +17,7 @@
  */
 package org.apache.gora.mongodb.store;
 
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.*;
 
 import java.io.IOException;
 
@@ -33,27 +33,36 @@ import org.junit.Test;
 
 import com.mongodb.BasicDBList;
 import com.mongodb.BasicDBObject;
+import java.util.Collections;
 
+import org.apache.avro.util.Utf8;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import static org.junit.Assert.assertNotNull;
 
 public abstract class TestMongoStore extends DataStoreTestBase {
 
+  private int keySequence;
+
   @Deprecated
   @Override
   protected DataStore<String, Employee> createEmployeeDataStore()
-      throws IOException {
+          throws IOException {
     throw new UnsupportedOperationException();
   }
 
   @Deprecated
   @Override
   protected DataStore<String, WebPage> createWebPageDataStore()
-      throws IOException {
+          throws IOException {
     throw new UnsupportedOperationException();
   }
 
   @Before
+  @Override
   public void setUp() throws Exception {
     super.setUp();
+    keySequence = 1;
   }
 
   public GoraMongodbTestDriver getTestDriver() {
@@ -84,7 +93,7 @@ public abstract class TestMongoStore extends DataStoreTestBase {
     BasicDBObject noField = new BasicDBObject();
     String field = "myField";
     Object item = store.fromMongoList(field, null, new BSONDecorator(noField),
-        null);
+            null);
     assertNotNull(item);
   }
 
@@ -94,7 +103,7 @@ public abstract class TestMongoStore extends DataStoreTestBase {
     String field = "myField";
     BasicDBObject emptyField = new BasicDBObject(field, new BasicDBList());
     Object item = store.fromMongoList(field, null,
-        new BSONDecorator(emptyField), null);
+            new BSONDecorator(emptyField), null);
     assertNotNull(item);
   }
 
@@ -104,7 +113,7 @@ public abstract class TestMongoStore extends DataStoreTestBase {
     BasicDBObject noField = new BasicDBObject();
     String field = "myField";
     Object item = store.fromMongoMap(field, null, new BSONDecorator(noField),
-        null);
+            null);
     assertNotNull(item);
   }
 
@@ -114,7 +123,62 @@ public abstract class TestMongoStore extends DataStoreTestBase {
     String field = "myField";
     BasicDBObject emptyField = new BasicDBObject(field, new BasicDBObject());
     Object item = store.fromMongoMap(field, null,
-        new BSONDecorator(emptyField), null);
+            new BSONDecorator(emptyField), null);
     assertNotNull(item);
   }
+
+  @Test
+  public void testResultProgress() throws Exception {
+    Query<String, WebPage> q;
+
+    // empty
+    q = webPageStore.newQuery();
+    assertProgress(q, 1);
+
+    addWebPage();
+
+    // one
+    q = webPageStore.newQuery();
+    assertProgress(q, 0, 1);
+
+    addWebPage();
+
+    // two
+    q = webPageStore.newQuery();
+    assertProgress(q, 0, 0.5f, 1);
+
+    addWebPage();
+
+    // three
+    q = webPageStore.newQuery();
+    assertProgress(q, 0, 0.33f, 0.66f, 1);
+  }
+
+  @Test
+  public void testResultProgressWithLimit() throws Exception {
+    for (int i = 0; i < 5; i++) {
+      addWebPage();
+    }
+    Query<String, WebPage> q = webPageStore.newQuery();
+    q.setLimit(2);
+
+    assertProgress(q, 0, 0.5f, 1);
+  }
+
+  private void assertProgress(Query<String, WebPage> q, float... progress) throws Exception
{
+    Result<String, WebPage> r = webPageStore.execute(q);
+    int i = 0;
+    do {
+      assertEquals(progress[i++], r.getProgress(), 0.01f);
+    } while (r.next());
+    r.close();
+  }
+
+  private void addWebPage() {
+    String key = String.valueOf(keySequence++);
+    WebPage p1 = webPageStore.newPersistent();
+    p1.setUrl(new Utf8(key));
+    p1.setHeaders(Collections.singletonMap((CharSequence) "header", (CharSequence) "value"));
+    webPageStore.put(key, p1);
+  }
 }


Mime
View raw message