geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jinmeil...@apache.org
Subject [08/32] incubator-geode git commit: GEODE-11: Pagination of Lucene Query Results after entries are destroyed from the region
Date Tue, 05 Jul 2016 23:15:31 GMT
GEODE-11: Pagination of Lucene Query Results after entries are destroyed from the region

If a region entry is destroyed after a query is executed, the pages.next() call returns
a null value for the deleted entry. The code is changed to ignore deleted entries and
fetch the next available entry so that the page count remains consistent. Added tests to
verify scenarios of single, multiple and all region entry deletions.

Added a test to verify that all entries are fetched in one page when pagination is
disabled.

Signed-off-by: Anilkumar Gingade <agingade@pivotal.io>

This closes #178


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/1a3c38af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/1a3c38af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/1a3c38af

Branch: refs/heads/feature/GEODE-1571
Commit: 1a3c38af10ad0b3dc0e6e9222a2c44f4f6598a3e
Parents: 6a9a83c
Author: Aparna Dharmakkan <adharmakkan@pivotal.io>
Authored: Wed Jun 29 16:06:38 2016 -0700
Committer: Dan Smith <upthewaterspout@apache.org>
Committed: Fri Jul 1 11:46:40 2016 -0700

----------------------------------------------------------------------
 .../PageableLuceneQueryResultsImpl.java         |  66 ++++++----
 .../lucene/LuceneQueriesIntegrationTest.java    | 119 +++++++++++++++++--
 2 files changed, 158 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1a3c38af/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/PageableLuceneQueryResultsImpl.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/PageableLuceneQueryResultsImpl.java
b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/PageableLuceneQueryResultsImpl.java
index 17668a7..1563df0 100644
--- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/PageableLuceneQueryResultsImpl.java
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/PageableLuceneQueryResultsImpl.java
@@ -20,6 +20,7 @@
 package com.gemstone.gemfire.cache.lucene.internal;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
@@ -42,7 +43,11 @@ public class PageableLuceneQueryResultsImpl<K,V> implements PageableLuceneQueryR
    *  list of docs matching search query
    */
   private final List<EntryScore<K>> hits;
-  
+
+  /**
+   * * Current page of results
+   */
+  private List<LuceneResultStruct<K,V>> currentPage;
   /**
    * The maximum score. Lazily evaluated
    */
@@ -69,38 +74,59 @@ public class PageableLuceneQueryResultsImpl<K,V> implements PageableLuceneQueryR
     this.pageSize = pageSize == 0 ? Integer.MAX_VALUE : pageSize;
   }
 
-  @Override
-  public List<LuceneResultStruct<K,V>> next() {
-    if(!hasNext()) {
-      throw new NoSuchElementException();
-    }
-    
-    int end = currentHit + pageSize;
-    end = end > hits.size() ? hits.size() : end;
-    List<EntryScore<K>> scores = hits.subList(currentHit, end);
-    
-    ArrayList<K> keys = new ArrayList<K>(hits.size());
+
+  public List<LuceneResultStruct<K,V>> getHitEntries(int fromIndex, int toIndex)
{
+    List<EntryScore<K>> scores = hits.subList(fromIndex, toIndex);
+    ArrayList<K> keys = new ArrayList<K>(scores.size());
     for(EntryScore<K> score : scores) {
       keys.add(score.getKey());
     }
-    
+
     Map<K,V> values = userRegion.getAll(keys);
 
-    ArrayList<LuceneResultStruct<K,V>> results = new ArrayList<LuceneResultStruct<K,V>>(hits.size());
+    ArrayList<LuceneResultStruct<K,V>> results = new ArrayList<LuceneResultStruct<K,V>>(scores.size());
     for(EntryScore<K> score : scores) {
       V value = values.get(score.getKey());
-      results.add(new LuceneResultStructImpl(score.getKey(), value, score.getScore()));
+      if (value!=null)
+        results.add(new LuceneResultStructImpl(score.getKey(), value, score.getScore()));
     }
-    
-
-    currentHit = end;
-    
     return results;
   }
 
   @Override
+  public List<LuceneResultStruct<K,V>> next() {
+    if(!hasNext()) {
+      throw new NoSuchElementException();
+    }
+    List<LuceneResultStruct<K,V>> result = advancePage();
+    currentPage = null;
+    return result;
+  }
+
+  private List<LuceneResultStruct<K, V>> advancePage() {
+    if(currentPage != null) {
+      return currentPage;
+    }
+
+    int resultSize = (pageSize != Integer.MAX_VALUE) ? pageSize : hits.size();
+    currentPage = new ArrayList<LuceneResultStruct<K,V>>(resultSize);
+    while (currentPage.size()<pageSize && currentHit < hits.size()) {
+      int end = currentHit + pageSize - currentPage.size();
+      end = end > hits.size() ? hits.size() : end;
+      currentPage.addAll(getHitEntries(currentHit, end));
+      currentHit = end;
+    }
+    return currentPage;
+  }
+
+  @Override
   public boolean hasNext() {
-    return hits.size() > currentHit;
+
+    advancePage();
+    if ( currentPage.isEmpty() ) {
+      return false;
+    }
+    return true;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1a3c38af/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
index 988e4f5..ccd6d41 100644
--- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
+++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.lucene.analysis.Analyzer;
@@ -35,6 +36,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.util.CharTokenizer;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.contrib.java.lang.system.SystemOutRule;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.ExpectedException;
 
@@ -43,6 +45,7 @@ import com.gemstone.gemfire.cache.RegionShortcut;
 import com.gemstone.gemfire.cache.lucene.test.TestObject;
 import com.gemstone.gemfire.pdx.JSONFormatter;
 import com.gemstone.gemfire.pdx.PdxInstance;
+import com.gemstone.gemfire.pdx.internal.AutoSerializableManager.ObjectArrayField;
 import com.gemstone.gemfire.test.junit.categories.IntegrationTest;
 
 /**
@@ -119,28 +122,32 @@ public class LuceneQueriesIntegrationTest extends LuceneIntegrationTest
{
   @Test()
   public void shouldPaginateResults() throws Exception {
 
-    final LuceneQuery<Object, Object> query = addValuesAndCreateQuery();
+    final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(2);
 
     final PageableLuceneQueryResults<Object, Object> pages = query.findPages();
     assertTrue(pages.hasNext());
-    assertEquals(3, pages.size());
+    assertEquals(7, pages.size());
     final List<LuceneResultStruct<Object, Object>> page1 = pages.next();
     final List<LuceneResultStruct<Object, Object>> page2 = pages.next();
+    final List<LuceneResultStruct<Object, Object>> page3 = pages.next();
+    final List<LuceneResultStruct<Object, Object>> page4 = pages.next();
     List<LuceneResultStruct<Object, Object>> allEntries=new ArrayList<>();
     allEntries.addAll(page1);
     allEntries.addAll(page2);
+    allEntries.addAll(page3);
+    allEntries.addAll(page4);
 
     assertEquals(region.keySet(), allEntries.stream().map(entry -> entry.getKey()).collect(Collectors.toSet()));
     assertEquals(region.values(), allEntries.stream().map(entry -> entry.getValue()).collect(Collectors.toSet()));
-
   }
+
   @Test
   public void shouldReturnValuesFromFindValues() throws Exception {
-    final LuceneQuery<Object, Object> query = addValuesAndCreateQuery();
+    final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(2);
     assertEquals(region.values(), new HashSet(query.findValues()));
   }
 
-  private LuceneQuery<Object, Object> addValuesAndCreateQuery() {
+  private LuceneQuery<Object, Object> addValuesAndCreateQuery(int pagesize) {
     luceneService.createIndex(INDEX_NAME, REGION_NAME, "field1", "field2");
     region = cache.createRegionFactory(RegionShortcut.PARTITION)
       .create(REGION_NAME);
@@ -153,10 +160,14 @@ public class LuceneQueriesIntegrationTest extends LuceneIntegrationTest
{
     region.put("A", new TestObject(value1, value1));
     region.put("B", new TestObject(value2, value2));
     region.put("C", new TestObject(value3, value3));
+    region.put("D", new TestObject(value1, value1));
+    region.put("E", new TestObject(value2, value2));
+    region.put("F", new TestObject(value3, value3));
+    region.put("G", new TestObject(value1, value2));
 
     index.waitUntilFlushed(60000);
     return luceneService.createLuceneQueryFactory()
-      .setPageSize(2)
+      .setPageSize(pagesize)
       .create(INDEX_NAME, REGION_NAME,
       "one", "field1");
   }
@@ -260,7 +271,101 @@ public class LuceneQueriesIntegrationTest extends LuceneIntegrationTest
{
     thrown.expect(LuceneQueryException.class);
     query.findPages();
   }
-  
+
+  @Test
+  public void shouldReturnAllResultsWhenPaginationIsDisabled() throws Exception {
+    // Pagination disabled by setting page size = 0.
+    final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(0);
+    final PageableLuceneQueryResults<Object, Object> pages = query.findPages();
+    assertTrue(pages.hasNext());
+    assertEquals(7, pages.size());
+    final List<LuceneResultStruct<Object, Object>> page = pages.next();
+    assertFalse(pages.hasNext());
+    assertEquals(region.keySet(), page.stream().map(entry -> entry.getKey()).collect(Collectors.toSet()));
+    assertEquals(region.values(), page.stream().map(entry -> entry.getValue()).collect(Collectors.toSet()));
+  }
+
+  @Test
+  public void shouldReturnCorrectResultsOnDeletionAfterQueryExecution() throws Exception
{
+    final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(2);
+    final PageableLuceneQueryResults<Object, Object> pages = query.findPages();
+    List<LuceneResultStruct<Object, Object>> allEntries=new ArrayList<>();
+    assertTrue(pages.hasNext());
+    assertEquals(7, pages.size());
+    // Destroying an entry from the region after the query is executed.
+    region.destroy("C");
+    final List<LuceneResultStruct<Object, Object>> page1 = pages.next();
+    assertEquals(2, page1.size());
+    final List<LuceneResultStruct<Object, Object>> page2 = pages.next();
+    assertEquals(2, page2.size());
+    final List<LuceneResultStruct<Object, Object>> page3 = pages.next();
+    assertEquals(2, page3.size());
+    assertFalse(pages.hasNext());
+
+    allEntries.addAll(page1);
+    allEntries.addAll(page2);
+    allEntries.addAll(page3);
+    assertEquals(region.keySet(), allEntries.stream().map(entry -> entry.getKey()).collect(Collectors.toSet()));
+    assertEquals(region.values(), allEntries.stream().map(entry -> entry.getValue()).collect(Collectors.toSet()));
+  }
+
+  @Test
+  public void shouldReturnCorrectResultsOnMultipleDeletionsAfterQueryExecution() throws Exception
{
+    final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(2);
+
+    final PageableLuceneQueryResults<Object, Object> pages = query.findPages();
+    List<LuceneResultStruct<Object, Object>> allEntries=new ArrayList<>();
+
+    assertTrue(pages.hasNext());
+    assertEquals(7, pages.size());
+
+    // Destroying an entry from the region after the query is executed.
+    region.destroy("C");
+    allEntries.addAll(pages.next());
+
+    // Destroying an entry from allEntries and the region after it is fetched through pages.next().
+    Object removeKey = ((LuceneResultStruct)allEntries.remove(0)).getKey();
+    region.destroy(removeKey);
+    allEntries.addAll(pages.next());
+
+    // Destroying a region entry which has't been fetched through pages.next() yet.
+    Set resultKeySet = allEntries.stream().map(entry -> entry.getKey()).collect(Collectors.toSet());
+
+    for(Object key : region.keySet()) {
+      if (!resultKeySet.contains(key)) {
+        region.destroy(key);
+        break;
+      }
+    }
+
+    allEntries.addAll(pages.next());
+    assertFalse(pages.hasNext());
+
+    assertEquals(region.keySet(), allEntries.stream().map(entry -> entry.getKey()).collect(Collectors.toSet()));
+    assertEquals(region.values(), allEntries.stream().map(entry -> entry.getValue()).collect(Collectors.toSet()));
+  }
+
+  @Test
+  public void shouldReturnCorrectResultsOnAllDeletionsAfterQueryExecution() throws Exception
{
+    final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(2);
+
+    final PageableLuceneQueryResults<Object, Object> pages = query.findPages();
+    assertTrue(pages.hasNext());
+    assertEquals(7, pages.size());
+    region.destroy("A");
+    region.destroy("B");
+    region.destroy("C");
+    region.destroy("D");
+    region.destroy("E");
+    region.destroy("F");
+    region.destroy("G");
+    assertTrue(pages.hasNext());
+    final List<LuceneResultStruct<Object, Object>> page1 = pages.next();
+    assertEquals(2, page1.size());
+    assertFalse(pages.hasNext());
+
+  }
+
   private PdxInstance insertAJson(Region region, String key) {
     String jsonCustomer = "{"
         + "\"name\": \""+key+"\","


Mime
View raw message