incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [3/3] git commit: Fixing a double search problem.
Date Thu, 02 Jan 2014 15:46:33 GMT
Fixing a double search problem.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/da26c980
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/da26c980
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/da26c980

Branch: refs/heads/master
Commit: da26c9804da11aa930e0ee164e43a4f3ee80f431
Parents: eda8a39
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Jan 2 10:45:00 2014 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Jan 2 10:46:25 2014 -0500

----------------------------------------------------------------------
 .../blur/thrift/ThriftBlurShardServer.java      |  8 ++++-
 .../java/org/apache/blur/utils/BlurUtil.java    |  4 +--
 .../org/apache/blur/thrift/BlurClusterTest.java | 33 ++++++++++++++++----
 .../src/main/resources/blur-default.properties  |  2 +-
 4 files changed, 36 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da26c980/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index d352dc1..44c3a0e 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -19,6 +19,7 @@ package org.apache.blur.thrift;
 import static org.apache.blur.utils.BlurConstants.BLUR_CLUSTER;
 import static org.apache.blur.utils.BlurConstants.BLUR_CLUSTER_NAME;
 import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_BIND_PORT;
+import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_REMOTE_FETCH_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT;
 import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT;
 import static org.apache.blur.utils.BlurConstants.BLUR_INDEXMANAGER_FACET_THREAD_COUNT;
@@ -212,7 +213,12 @@ public class ThriftBlurShardServer extends ThriftServer {
     BooleanQuery.setMaxClauseCount(configuration.getInt(BLUR_MAX_CLAUSE_COUNT, 1024));
 
     int maxHeapPerRowFetch = configuration.getInt(BLUR_MAX_HEAP_PER_ROW_FETCH, 10000000);
-    int fetchCount = configuration.getInt(BLUR_SHARD_FETCHCOUNT, 100);
+    int remoteFetchCount = configuration.getInt(BLUR_CONTROLLER_REMOTE_FETCH_COUNT, 100);
+    int fetchCount = configuration.getInt(BLUR_SHARD_FETCHCOUNT, 110);
+    if (fetchCount + 1 <= remoteFetchCount) {
+      LOG.warn("[" + BLUR_SHARD_FETCHCOUNT + "] [" + fetchCount + "] is equal to or less
than ["
+          + BLUR_CONTROLLER_REMOTE_FETCH_COUNT + "] [" + remoteFetchCount + "], should be
at least 1 greater.");
+    }
     int indexManagerThreadCount = configuration.getInt(BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT,
32);
     int mutateThreadCount = configuration.getInt(BLUR_INDEXMANAGER_MUTATE_THREAD_COUNT, 32);
     int facetThreadCount = configuration.getInt(BLUR_INDEXMANAGER_FACET_THREAD_COUNT, 16);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da26c980/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index ee954a8..1c521df 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -443,11 +443,9 @@ public class BlurUtil {
     results.setShardInfo(hitsIterable.getShardInfo());
     if (query.minimumNumberOfResults > 0) {
       hitsIterable.skipTo(query.start);
-      int count = 0;
       BlurIterator<BlurResult, BlurException> iterator = hitsIterable.iterator();
-      while (iterator.hasNext() && count < query.fetch) {
+      for (int count = 0; count < query.fetch && iterator.hasNext(); count++)
{
         results.addToResults(iterator.next());
-        count++;
       }
     }
     if (results.results == null) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da26c980/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
index 1055385..66d4372 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
@@ -78,7 +78,7 @@ public class BlurClusterTest {
 
   private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp_BlurClusterTest"));
   private static MiniCluster miniCluster;
-  
+
   private int numberOfDocs = 1000;
 
   @BeforeClass
@@ -117,20 +117,39 @@ public class BlurClusterTest {
   @Test
   public void runClusterIntegrationTests() throws BlurException, TException, IOException,
InterruptedException,
       KeeperException {
+    start("testCreateTable");
     testCreateTable();
+    start("testLoadTable");
     testLoadTable();
+    start("testForEmptySchema");
     testForEmptySchema();
+    start("testQueryWithSelector");
     testQueryWithSelector();
+    start("testQueryWithFacets");
     testQueryWithFacets();
+    start("testBatchFetch");
     testBatchFetch();
+    start("testLoadTable");
     testQueryCancel();
+    start("testBackPressureViaQuery");
     testBackPressureViaQuery();
+    start("testTestShardFailover");
     testTestShardFailover();
+    start("testTermsList");
     testTermsList();
+    start("testCreateDisableAndRemoveTable");
     testCreateDisableAndRemoveTable();
+    start("testCreateTableWithCustomType");
     testCreateTableWithCustomType();
   }
 
+  private void start(String name) {
+    System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
+    System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
+    System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
+    System.out.println("Staring [" + name + "]");
+  }
+
   private void testForEmptySchema() throws BlurException, TException, IOException {
     Blur.Iface client = getClient();
     Schema schema = client.schema("test");
@@ -379,6 +398,13 @@ public class BlurClusterTest {
           client.query("test", blurQueryRow);
           fail.set(true);
         } catch (BlurException e) {
+          System.out.println("-------------------");
+          System.out.println("-------------------");
+          System.out.println("-------------------");
+          e.printStackTrace();
+          System.out.println("-------------------");
+          System.out.println("-------------------");
+          System.out.println("-------------------");
           error.set(e);
         } catch (TException e) {
           e.printStackTrace();
@@ -446,11 +472,6 @@ public class BlurClusterTest {
   public void testTestShardFailover() throws BlurException, TException, InterruptedException,
IOException,
       KeeperException {
 
-    System.out.println("===========================");
-    System.out.println("===========================");
-    System.out.println("===========================");
-    System.out.println("===========================");
-
     Iface client = getClient();
     BlurQuery blurQuery = new BlurQuery();
     blurQuery.setUseCacheIfPresent(false);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/da26c980/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 6cd6904..8f09e3b 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -167,7 +167,7 @@ blur.shard.internal.search.thread.count=8
 blur.shard.warmup.thread.count=8
 
 # The fetch count per Lucene search, this fetches pointers to hits
-blur.shard.fetchcount=100
+blur.shard.fetchcount=110
 
 # Heap limit on row fetch, once this limit has been reached the request will return
 blur.max.heap.per.row.fetch=10000000


Mime
View raw message