incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Fixing issues with cache when the layout of the cluster would change due to a shard server failure. The cluster test was flawed because it disabled the use of cached results. Also improved the error message when the controller can not compl
Date Sun, 01 Mar 2015 18:56:14 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master e0a6a106e -> 5f16374ef


Fixing issues with cache when the layout of the cluster would change due to a shard server
failure.  The cluster test was flawed because it disabled the use of cached results.  Also
improved the error message when the controller can not complete a query due to shard server
errors.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/5f16374e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/5f16374e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/5f16374e

Branch: refs/heads/master
Commit: 5f16374efdbdd1cb3c62a167a10e4c859a04af1b
Parents: e0a6a10
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sun Mar 1 13:56:10 2015 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sun Mar 1 13:56:10 2015 -0500

----------------------------------------------------------------------
 .../org/apache/blur/server/cache/ClassObj.java  | 51 ++++++++++++
 .../apache/blur/server/cache/ThriftCache.java   | 38 ++++++---
 .../blur/server/cache/ThriftCacheKey.java       | 54 ++++++++----
 .../blur/server/cache/ThriftCacheServer.java    | 48 ++++++++---
 .../blur/server/cache/ThriftCacheValue.java     | 35 +++++++-
 .../blur/thrift/BlurControllerServer.java       |  5 +-
 .../thrift/QueryCoundNotBeCompletedReport.java  | 81 ++++++++++++++++++
 .../blur/thrift/ThriftBlurShardServer.java      |  2 +-
 .../blur/server/cache/ThriftCacheKeyTest.java   | 86 +++++++++++++++-----
 .../server/cache/ThriftCacheServerTest.java     | 54 +++++++++++-
 .../blur/server/cache/ThriftCacheTest.java      |  6 +-
 .../apache/blur/thrift/BlurClusterTestBase.java |  1 -
 12 files changed, 393 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java b/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java
new file mode 100644
index 0000000..eabcc7b
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/server/cache/ClassObj.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.server.cache;
+
+public class ClassObj<T> {
+
+  private final Class<T> _clazz;
+
+  public ClassObj(Class<T> clazz) {
+    if (clazz == null) {
+      throw new RuntimeException("Class cannot be null.");
+    }
+    _clazz = clazz;
+  }
+
+  public Class<T> getClazz() {
+    return _clazz;
+  }
+
+  @Override
+  public int hashCode() {
+    return _clazz.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    ClassObj<?> other = (ClassObj<?>) obj;
+    return _clazz == other._clazz;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java
index e4b428c..7093a6a 100644
--- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java
+++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCache.java
@@ -25,6 +25,8 @@ import static org.apache.blur.metrics.MetricsConstants.THRIFT_CACHE;
 import java.util.Iterator;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -36,8 +38,8 @@ import org.apache.blur.user.User;
 import org.apache.blur.user.UserContext;
 
 import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+import com.googlecode.concurrentlinkedhashmap.EntryWeigher;
 import com.googlecode.concurrentlinkedhashmap.EvictionListener;
-import com.googlecode.concurrentlinkedhashmap.Weigher;
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Meter;
 import com.yammer.metrics.core.MetricName;
@@ -47,6 +49,7 @@ public class ThriftCache {
   private static final Log LOG = LogFactory.getLog(ThriftCache.class);
 
   private final ConcurrentLinkedHashMap<ThriftCacheKey<?>, ThriftCacheValue<?>>
_cacheMap;
+  private final ConcurrentMap<String, Long> _lastModTimestamps = new ConcurrentHashMap<String,
Long>();
   private final Meter _hits;
   private final Meter _misses;
   private final Meter _evictions;
@@ -59,10 +62,10 @@ public class ThriftCache {
     _misses = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, THRIFT_CACHE, MISS), MISS,
TimeUnit.SECONDS);
     _evictions = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, THRIFT_CACHE, EVICTION),
EVICTION, TimeUnit.SECONDS);
     _cacheMap = new ConcurrentLinkedHashMap.Builder<ThriftCacheKey<?>, ThriftCacheValue<?>>()
-        .weigher(new Weigher<ThriftCacheValue<?>>() {
+        .weigher(new EntryWeigher<ThriftCacheKey<?>, ThriftCacheValue<?>>()
{
           @Override
-          public int weightOf(ThriftCacheValue<?> value) {
-            return value.size();
+          public int weightOf(ThriftCacheKey<?> key, ThriftCacheValue<?> value)
{
+            return key.size() + value.size();
           }
         }).listener(new EvictionListener<ThriftCacheKey<?>, ThriftCacheValue<?>>()
{
           @Override
@@ -77,6 +80,16 @@ public class ThriftCache {
   }
 
   public <K extends TBase<?, ?>, V extends TBase<?, ?>> V put(ThriftCacheKey<K>
key, V t) throws BlurException {
+    synchronized (_lastModTimestamps) {
+      Long lastModTimestamp = _lastModTimestamps.get(key.getTable());
+      if (lastModTimestamp != null && key.getTimestamp() < lastModTimestamp) {
+        // This means that the key was created before the index was modified. So
+        // do not cache the value because it's already out of date with the
+        // index.
+        return t;
+      }
+    }
+    LOG.debug("Inserting into cache [{0}] with key [{1}]", t, key);
     _cacheMap.put(key, new ThriftCacheValue<V>(t));
     return t;
   }
@@ -97,17 +110,15 @@ public class ThriftCache {
     return value.getValue(clazz);
   }
 
-  public <K extends TBase<?, ?>> ThriftCacheKey<K> getKey(String table,
K tkey, Class<K> clazz) {
+  public <K extends TBase<?, ?>> ThriftCacheKey<K> getKey(String table,
int[] shards, K tkey, Class<K> clazz) throws BlurException {
     User user = UserContext.getUser();
-    return new ThriftCacheKey<K>(user, table, tkey, clazz);
-  }
-
-  public void clear() {
-    LOG.info("Clearing all cache.");
-    _cacheMap.clear();
+    return new ThriftCacheKey<K>(user, table, shards, tkey, clazz);
   }
 
   public void clearTable(String table) {
+    synchronized (_lastModTimestamps) {
+      _lastModTimestamps.put(table, System.nanoTime());
+    }
     LOG.info("Clearing cache for table [{0}]", table);
     Set<Entry<ThriftCacheKey<?>, ThriftCacheValue<?>>> entrySet =
_cacheMap.entrySet();
     Iterator<Entry<ThriftCacheKey<?>, ThriftCacheValue<?>>> iterator
= entrySet.iterator();
@@ -119,6 +130,11 @@ public class ThriftCache {
     }
   }
 
+  public void clear() {
+    LOG.info("Clearing all cache.");
+    _cacheMap.clear();
+  }
+
   public long size() {
     return _cacheMap.weightedSize();
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java
index 790533b..5866966 100644
--- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java
+++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheKey.java
@@ -16,10 +16,12 @@
  */
 package org.apache.blur.server.cache;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.TreeMap;
 
 import org.apache.blur.thirdparty.thrift_0_9_0.TBase;
+import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.user.User;
 
 public class ThriftCacheKey<T extends TBase<?, ?>> {
@@ -27,11 +29,16 @@ public class ThriftCacheKey<T extends TBase<?, ?>> {
   private final String _username;
   private final Map<String, String> _attributes;
   private final String _table;
-  private final T _t;
-  private final String _clazz;
+  private final int[] _shards;
+  private final ClassObj<T> _clazz;
+  private final ThriftCacheValue<T> _key;
 
-  public ThriftCacheKey(User user, String table, T t, Class<T> clazz) {
-    _clazz = clazz.getName();
+  // not apart of the key for equally
+  private final transient long _timestamp;
+
+  public ThriftCacheKey(User user, String table, int[] shards, T t, Class<T> clazz)
throws BlurException {
+    _timestamp = System.nanoTime();
+    _clazz = new ClassObj<T>(clazz);
     if (user != null) {
       _username = user.getUsername();
       Map<String, String> attributes = user.getAttributes();
@@ -45,7 +52,12 @@ public class ThriftCacheKey<T extends TBase<?, ?>> {
       _attributes = null;
     }
     _table = table;
-    _t = t;
+    _shards = shards;
+    _key = new ThriftCacheValue<T>(t);
+  }
+
+  public long getTimestamp() {
+    return _timestamp;
   }
 
   public String getTable() {
@@ -53,12 +65,28 @@ public class ThriftCacheKey<T extends TBase<?, ?>> {
   }
 
   @Override
+  public String toString() {
+    try {
+      return "ThriftCacheKey [_username=" + _username + ", _attributes=" + _attributes +
", _table=" + _table
+          + ", _shards=" + Arrays.toString(_shards) + ", _clazz=" + _clazz + ", _key="
+          + _key.getValue(_clazz.getClazz()) + ", _timestamp=" + _timestamp + "]";
+    } catch (BlurException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public int size() {
+    return _key.size();
+  }
+
+  @Override
   public int hashCode() {
     final int prime = 31;
     int result = 1;
     result = prime * result + ((_attributes == null) ? 0 : _attributes.hashCode());
     result = prime * result + ((_clazz == null) ? 0 : _clazz.hashCode());
-    result = prime * result + ((_t == null) ? 0 : _t.hashCode());
+    result = prime * result + ((_key == null) ? 0 : _key.hashCode());
+    result = prime * result + Arrays.hashCode(_shards);
     result = prime * result + ((_table == null) ? 0 : _table.hashCode());
     result = prime * result + ((_username == null) ? 0 : _username.hashCode());
     return result;
@@ -83,10 +111,12 @@ public class ThriftCacheKey<T extends TBase<?, ?>> {
         return false;
     } else if (!_clazz.equals(other._clazz))
       return false;
-    if (_t == null) {
-      if (other._t != null)
+    if (_key == null) {
+      if (other._key != null)
         return false;
-    } else if (!_t.equals(other._t))
+    } else if (!_key.equals(other._key))
+      return false;
+    if (!Arrays.equals(_shards, other._shards))
       return false;
     if (_table == null) {
       if (other._table != null)
@@ -101,10 +131,4 @@ public class ThriftCacheKey<T extends TBase<?, ?>> {
     return true;
   }
 
-  @Override
-  public String toString() {
-    return "ThriftCacheKey [_username=" + _username + ", _attributes=" + _attributes + ",
_table=" + _table + ", _t="
-        + _t + ", _clazz=" + _clazz + "]";
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java
index dbd395f..7264801 100644
--- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java
+++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheServer.java
@@ -16,15 +16,20 @@
  */
 package org.apache.blur.server.cache;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 
 import org.apache.blur.BlurConfiguration;
+import org.apache.blur.manager.IndexServer;
 import org.apache.blur.server.FilteredBlurServer;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.BException;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.BlurQuery;
@@ -32,19 +37,23 @@ import org.apache.blur.thrift.generated.BlurResults;
 import org.apache.blur.thrift.generated.FetchResult;
 import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.TableStats;
+import org.apache.blur.utils.ShardUtil;
 
 public class ThriftCacheServer extends FilteredBlurServer {
 
   private final ThriftCache _thriftCache;
+  private final IndexServer _indexServer;
 
-  public ThriftCacheServer(BlurConfiguration configuration, Iface iface, ThriftCache thriftCache)
{
+  public ThriftCacheServer(BlurConfiguration configuration, Iface iface, IndexServer indexServer,
+      ThriftCache thriftCache) {
     super(configuration, iface, true);
     _thriftCache = thriftCache;
+    _indexServer = indexServer;
   }
 
   @Override
   public TableStats tableStats(String table) throws BlurException, TException {
-    ThriftCacheKey<TableStats> key = _thriftCache.getKey(table, null, TableStats.class);
+    ThriftCacheKey<TableStats> key = _thriftCache.getKey(table, getShards(table), null,
TableStats.class);
     TableStats results = _thriftCache.get(key, TableStats.class);
     if (results != null) {
       return results;
@@ -52,6 +61,22 @@ public class ThriftCacheServer extends FilteredBlurServer {
     return _thriftCache.put(key, super.tableStats(table));
   }
 
+  private int[] getShards(String table) throws BlurException {
+    try {
+      Set<String> keySet = _indexServer.getIndexes(table).keySet();
+      int[] shards = new int[keySet.size()];
+      int i = 0;
+      for (String s : keySet) {
+        int shardIndex = ShardUtil.getShardIndex(s);
+        shards[i++] = shardIndex;
+      }
+      Arrays.sort(shards);
+      return shards;
+    } catch (IOException e) {
+      throw new BException("Unknown error while trying to get current shards for table [{0}]",
e, table);
+    }
+  }
+
   @Override
   public BlurResults query(String table, BlurQuery blurQuery) throws BlurException, TException
{
     boolean useCacheIfPresent = blurQuery.isUseCacheIfPresent();
@@ -66,14 +91,16 @@ public class ThriftCacheServer extends FilteredBlurServer {
     copy.userContext = null;
     copy.cacheResult = false;
     copy.startTime = 0;
-    ThriftCacheKey<BlurQuery> key = _thriftCache.getKey(table, copy, BlurQuery.class);
-    BlurResults results = _thriftCache.get(key, BlurResults.class);
-    if (results != null && useCacheIfPresent) {
-      return results;
+    ThriftCacheKey<BlurQuery> key = _thriftCache.getKey(table, getShards(table), copy,
BlurQuery.class);
+    if (useCacheIfPresent) {
+      BlurResults results = _thriftCache.get(key, BlurResults.class);
+      if (results != null) {
+        return results;
+      }
     }
     BlurResults blurResults = super.query(table, blurQuery);
     if (cacheResult) {
-      return _thriftCache.put(key, blurResults);  
+      return _thriftCache.put(key, blurResults);
     }
     return blurResults;
   }
@@ -81,7 +108,7 @@ public class ThriftCacheServer extends FilteredBlurServer {
   @Override
   public FetchResult fetchRow(String table, Selector selector) throws BlurException, TException
{
     Selector copy = new Selector(selector);
-    ThriftCacheKey<Selector> key = _thriftCache.getKey(table, copy, Selector.class);
+    ThriftCacheKey<Selector> key = _thriftCache.getKey(table, getShards(table), copy,
Selector.class);
     FetchResult results = _thriftCache.get(key, FetchResult.class);
     if (results != null) {
       return results;
@@ -101,7 +128,7 @@ public class ThriftCacheServer extends FilteredBlurServer {
     for (int i = 0; i < selectors.size(); i++) {
       Selector selector = selectors.get(i);
       Selector copy = new Selector(selector);
-      ThriftCacheKey<Selector> key = _thriftCache.getKey(table, copy, Selector.class);
+      ThriftCacheKey<Selector> key = _thriftCache.getKey(table, getShards(table), copy,
Selector.class);
       FetchResult fetchResult = _thriftCache.get(key, FetchResult.class);
       if (fetchResult != null) {
         resultMap.put(i, fetchResult);
@@ -117,7 +144,8 @@ public class ThriftCacheServer extends FilteredBlurServer {
       for (int i = 0; i < missingResults.size(); i++) {
         Selector selector = selectorRequest.get(i);
         FetchResult fetchResult = missingResults.get(i);
-        ThriftCacheKey<Selector> key = _thriftCache.getKey(table, new Selector(selector),
Selector.class);
+        ThriftCacheKey<Selector> key = _thriftCache.getKey(table, getShards(table),
new Selector(selector),
+            Selector.class);
         _thriftCache.put(key, fetchResult);
         int originalIndex = requestMapping.get(i);
         resultMap.put(originalIndex, fetchResult);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java
index 7d259f4..84d5606 100644
--- a/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java
+++ b/blur-core/src/main/java/org/apache/blur/server/cache/ThriftCacheValue.java
@@ -16,6 +16,8 @@
  */
 package org.apache.blur.server.cache;
 
+import java.util.Arrays;
+
 import org.apache.blur.thirdparty.thrift_0_9_0.TBase;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TCompactProtocol;
@@ -29,8 +31,12 @@ public class ThriftCacheValue<T extends TBase<?, ?>> {
   private final byte[] _data;
 
   public ThriftCacheValue(T t) throws BlurException {
+    _data = toBytes(t);
+  }
+
+  public static <T extends TBase<?, ?>> byte[] toBytes(T t) throws BException
{
     if (t == null) {
-      _data = null;
+      return null;
     } else {
       TMemoryBuffer transport = new TMemoryBuffer(1024);
       try {
@@ -38,11 +44,11 @@ public class ThriftCacheValue<T extends TBase<?, ?>> {
       } catch (TException e) {
         throw new BException("Unknown error while trying to read from cache.", e);
       }
-      _data = trim(transport.getArray(), transport.length());
+      return trim(transport.getArray(), transport.length());
     }
   }
 
-  private byte[] trim(byte[] bs, int len) {
+  public static byte[] trim(byte[] bs, int len) {
     if (bs.length == len) {
       return bs;
     }
@@ -75,4 +81,27 @@ public class ThriftCacheValue<T extends TBase<?, ?>> {
       throw new BException("Unknown error while trying to read from cache.", e);
     }
   }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + Arrays.hashCode(_data);
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    ThriftCacheValue<?> other = (ThriftCacheValue<?>) obj;
+    if (!Arrays.equals(_data, other._data))
+      return false;
+    return true;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
index a29867b..4c45a90 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
@@ -475,6 +475,7 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
 
       BlurUtil.setStartTime(blurQuery);
 
+      QueryCoundNotBeCompletedReport report = null;
       OUTER: for (int retries = 0; retries < _maxDefaultRetries; retries++) {
         Tracer selectorTrace = Trace.trace("selector - setup", Trace.param("retries", retries));
         final AtomicLongArray facetCounts = BlurUtil.getAtomicLongArraySameLengthAsList(blurQuery.facets);
@@ -534,6 +535,8 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
           if (!validResults(results, shardCount, blurQuery)) {
             BlurClientManager.sleep(_defaultDelay, _maxDefaultDelay, retries, _maxDefaultRetries);
             Map<String, String> map = getTableLayout(table);
+            Map<String, Long> shardInfo = results.getShardInfo();
+            report = new QueryCoundNotBeCompletedReport(map, shardCount, shardInfo, blurQuery);
             LOG.info("Current layout for table [{0}] is [{1}]", table, map);
             continue OUTER;
           }
@@ -544,7 +547,7 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
           }
         }
       }
-      throw new BException("Query could not be completed.");
+      throw new BException("Query could not be completed, report [{0}].", report);
     } catch (Exception e) {
       LOG.error("Unknown error during search of [table={0},blurQuery={1}]", e, table, blurQuery);
       if (e instanceof BlurException) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java
b/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java
new file mode 100644
index 0000000..0c62724
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/thrift/QueryCoundNotBeCompletedReport.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.thrift;
+
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class QueryCoundNotBeCompletedReport {
+
+  private final String _message;
+
+  public QueryCoundNotBeCompletedReport(Map<String, String> tableLayout, int shardCount,
Map<String, Long> shardInfo,
+      BlurQuery blurQuery) throws JSONException {
+    String missingShardsFromResult = "\n";
+    if (shardCount != shardInfo.size()) {
+
+      Set<String> missingShardResults = new TreeSet<String>();
+      Set<String> servers = new TreeSet<String>();
+      for (Entry<String, String> e : tableLayout.entrySet()) {
+        String shard = e.getKey();
+        String server = e.getValue();
+        if (!shardInfo.containsKey(shard)) {
+          missingShardResults.add(shard);
+          servers.add(server);
+        }
+      }
+      missingShardsFromResult = "\nThere appears to be some missing shards from the results.\n";
+      missingShardsFromResult += "Missing Shards:\n\t" + missingShardResults + "\n";
+      missingShardsFromResult += "From Servers:\n\t" + servers + "\n";
+    }
+
+    JSONObject jsonObject = new JSONObject();
+    jsonObject.put("tableLayout", getTableLayout(tableLayout));
+    jsonObject.put("tableShardCount", shardCount);
+    jsonObject.put("queryShardCount", shardInfo.size());
+    jsonObject.put("queryResultCount", getShardInfo(shardInfo));
+    _message = missingShardsFromResult + jsonObject.toString(1);
+  }
+
+  private JSONObject getShardInfo(Map<String, Long> shardInfo) throws JSONException
{
+    JSONObject j = new JSONObject();
+    for (Entry<String, Long> e : shardInfo.entrySet()) {
+      j.put(e.getKey(), e.getValue());
+    }
+    return j;
+  }
+
+  private JSONObject getTableLayout(Map<String, String> tableLayout) throws JSONException
{
+    JSONObject j = new JSONObject();
+    for (Entry<String, String> e : tableLayout.entrySet()) {
+      j.put(e.getKey(), e.getValue());
+    }
+    return j;
+  }
+
+  @Override
+  public String toString() {
+    return _message;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 545bf48..d8f15b0 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -289,7 +289,7 @@ public class ThriftBlurShardServer extends ThriftServer {
     List<ServerSecurityFilter> serverSecurity = getServerSecurityList(configuration,
         ServerSecurityFilterFactory.ServerType.SHARD);
 
-    Iface iface = new ThriftCacheServer(configuration, shardServer, thriftCache);
+    Iface iface = new ThriftCacheServer(configuration, shardServer, indexServer, thriftCache);
     iface = BlurUtil.wrapFilteredBlurServer(configuration, iface, true);
     iface = ServerSecurityUtil.applySecurity(iface, serverSecurity, true);
     iface = BlurUtil.recordMethodCallsAndAverageTimes(iface, Iface.class, false);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java
b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java
index 0ca81a6..8c5da28 100644
--- a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java
+++ b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheKeyTest.java
@@ -16,11 +16,15 @@
  */
 package org.apache.blur.server.cache;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
 
+import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.BlurQuery;
 import org.apache.blur.user.User;
 import org.junit.Test;
@@ -28,92 +32,122 @@ import org.junit.Test;
 public class ThriftCacheKeyTest {
 
   @Test
-  public void test1() {
+  public void test1() throws BlurException {
     User user = null;
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table,
bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table,
bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table,
new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table,
new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertEquals(key1, key2);
     assertEquals(key1.hashCode(), key2.hashCode());
   }
 
   @Test
-  public void test2() {
+  public void test2() throws BlurException {
     User user = null;
     String table = "t";
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table,
null, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table,
null, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table,
new int[] { 0, 1 }, null,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table,
new int[] { 0, 1 }, null,
+        BlurQuery.class);
 
     assertEquals(key1, key2);
     assertEquals(key1.hashCode(), key2.hashCode());
   }
 
   @Test
-  public void test3() {
+  public void test3() throws BlurException {
     User user = new User("test", null);
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table,
bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table,
bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table,
new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table,
new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertEquals(key1, key2);
     assertEquals(key1.hashCode(), key2.hashCode());
   }
 
   @Test
-  public void test4() {
+  public void test4() throws BlurException {
     User user = new User("test", map("a", "b"));
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table,
bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table,
bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user, table,
new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user, table,
new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertEquals(key1, key2);
     assertEquals(key1.hashCode(), key2.hashCode());
   }
 
   @Test
-  public void test5() {
+  public void test5a() throws BlurException {
     User user1 = new User("test1", null);
     User user2 = new User("test2", null);
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table,
bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table,
bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table,
new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table,
new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertFalse(key1.equals(key2));
     assertFalse(key1.hashCode() == key2.hashCode());
   }
 
   @Test
-  public void test6() {
+  public void test5b() throws BlurException {
+    User user1 = new User("test1", null);
+    User user2 = new User("test1", null);
+    String table = "t";
+    BlurQuery bq1 = new BlurQuery();
+    BlurQuery bq2 = new BlurQuery();
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table,
new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table,
new int[] { 0, 2 }, bq2,
+        BlurQuery.class);
+
+    assertFalse(key1.equals(key2));
+    assertFalse(key1.hashCode() == key2.hashCode());
+  }
+
+  @Test
+  public void test6() throws BlurException {
     User user1 = new User("test1", map("a", "b"));
     User user2 = new User("test1", map("a", "c"));
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table,
bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table,
bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table,
new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table,
new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertFalse(key1.equals(key2));
     assertFalse(key1.hashCode() == key2.hashCode());
   }
 
   @Test
-  public void test7() {
+  public void test7() throws BlurException {
     User user1 = new User("test1", map("a", "b"));
     User user2 = new User("test2", map("a", "b"));
     String table = "t";
     BlurQuery bq1 = new BlurQuery();
     BlurQuery bq2 = new BlurQuery();
-    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table,
bq1, BlurQuery.class);
-    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table,
bq2, BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key1 = new ThriftCacheKey<BlurQuery>(user1, table,
new int[] { 0, 1 }, bq1,
+        BlurQuery.class);
+    ThriftCacheKey<BlurQuery> key2 = new ThriftCacheKey<BlurQuery>(user2, table,
new int[] { 0, 1 }, bq2,
+        BlurQuery.class);
 
     assertFalse(key1.equals(key2));
     assertFalse(key1.hashCode() == key2.hashCode());
@@ -132,4 +166,12 @@ public class ThriftCacheKeyTest {
     }
     return map;
   }
+
+  public static SortedSet<String> getShards(String... shards) {
+    SortedSet<String> set = new TreeSet<String>();
+    for (String s : shards) {
+      set.add(s);
+    }
+    return set;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java
b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java
index 33accc3..587f1fb 100644
--- a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java
+++ b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheServerTest.java
@@ -22,11 +22,15 @@ import static org.junit.Assert.assertFalse;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedSet;
 
 import org.apache.blur.BlurConfiguration;
+import org.apache.blur.manager.IndexServer;
+import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thrift.generated.Arguments;
 import org.apache.blur.thrift.generated.Blur.Iface;
@@ -65,7 +69,7 @@ public class ThriftCacheServerTest {
   public void setup() throws IOException {
     _configuration = new BlurConfiguration();
     _thriftCache = new ThriftCache(10000);
-    _thriftCacheServer = new ThriftCacheServer(_configuration, getMock(), _thriftCache);
+    _thriftCacheServer = new ThriftCacheServer(_configuration, getMock(), getMockIndexServer(),
_thriftCache);
     _table = "t";
   }
 
@@ -507,4 +511,52 @@ public class ThriftCacheServerTest {
       }
     };
   }
+
+  private IndexServer getMockIndexServer() {
+    return new IndexServer() {
+
+      @Override
+      public long getTableSize(String table) throws IOException {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public Map<String, ShardState> getShardState(String table) {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public SortedSet<String> getShardListCurrentServerOnly(String table) throws IOException
{
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public long getRowCount(String table) throws IOException {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public long getRecordCount(String table) throws IOException {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public String getNodeName() {
+        throw new RuntimeException("Not implemented.");
+      }
+
+      @Override
+      public Map<String, BlurIndex> getIndexes(String table) throws IOException {
+        Map<String, BlurIndex> map = new HashMap<String, BlurIndex>();
+        map.put("shard-000000", null);
+        map.put("shard-000001", null);
+        return map;
+      }
+
+      @Override
+      public void close() throws IOException {
+        throw new RuntimeException("Not implemented.");
+      }
+    };
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java
index da96ba5..6fe8e8c 100644
--- a/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java
+++ b/blur-core/src/test/java/org/apache/blur/server/cache/ThriftCacheTest.java
@@ -33,7 +33,7 @@ public class ThriftCacheTest {
   public void test1() throws BlurException {
     ThriftCache thriftCache = new ThriftCache(10000);
     BlurQuery blurQuery = new BlurQuery();
-    ThriftCacheKey<?> key = thriftCache.getKey("t", blurQuery, BlurQuery.class);
+    ThriftCacheKey<?> key = thriftCache.getKey("t", new int[] { 0, 1 }, blurQuery,
BlurQuery.class);
     assertNull(thriftCache.get(key, BlurResults.class));
     BlurResults value = new BlurResults();
     // assert same instance
@@ -46,7 +46,7 @@ public class ThriftCacheTest {
   @Test
   public void test2() throws BlurException {
     ThriftCache thriftCache = new ThriftCache(10000);
-    ThriftCacheKey<?> key = thriftCache.getKey("t", null, TableStats.class);
+    ThriftCacheKey<?> key = thriftCache.getKey("t", new int[] { 0, 1 }, null, TableStats.class);
     assertNull(thriftCache.get(key, TableStats.class));
     TableStats value = new TableStats();
     // assert same instance
@@ -64,7 +64,7 @@ public class ThriftCacheTest {
       BlurQuery blurQuery = new BlurQuery();
       // just make keys different
       blurQuery.fetch = i;
-      ThriftCacheKey<?> key = thriftCache.getKey("t", blurQuery, BlurQuery.class);
+      ThriftCacheKey<?> key = thriftCache.getKey("t", new int[] { 0, 1 }, blurQuery,
BlurQuery.class);
       assertNull(thriftCache.get(key, BlurResults.class));
       BlurResults value = new BlurResults();
       // assert same instance

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5f16374e/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
index dd95513..335ae21 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
@@ -792,7 +792,6 @@ public abstract class BlurClusterTestBase {
     loadTable(tableName);
     Iface client = getClient();
     BlurQuery blurQuery = new BlurQuery();
-    blurQuery.setUseCacheIfPresent(false);
     Query query = new Query();
     query.setQuery("test.test:value");
     blurQuery.setQuery(query);


Mime
View raw message