incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/3] git commit: Fixed BLUR-260
Date Wed, 09 Oct 2013 17:42:10 GMT
Fixed BLUR-260


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a97d9790
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a97d9790
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a97d9790

Branch: refs/heads/apache-blur-0.2
Commit: a97d97906bca7230e56334f4b58684725575739f
Parents: 82072ff
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Oct 9 13:33:53 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Oct 9 13:33:53 2013 -0400

----------------------------------------------------------------------
 .../clusterstatus/ZookeeperPathConstants.java   |  4 ++
 .../DistributedLayoutFactoryImpl.java           | 42 ++++++++++++++++++
 .../MasterBasedDistributedLayoutFactory.java    | 45 ++++++++++++--------
 .../blur/thrift/BlurControllerServer.java       | 38 +++++++++++++----
 .../blur/thrift/ThriftBlurControllerServer.java |  1 +
 .../blur/thrift/ThriftBlurShardServer.java      | 33 +++++++-------
 .../org/apache/blur/thrift/BlurClusterTest.java | 18 ++++++--
 .../org/apache/blur/utils/BlurConstants.java    |  1 +
 8 files changed, 135 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a97d9790/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
index 9ddf7d8..3e1f619 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
@@ -99,4 +99,8 @@ public class ZookeeperPathConstants {
     return getTablePath(cluster, table) + "/precache";
   }
 
+  public static String getShardLayoutPath(String cluster) {
+    return getClusterPath(cluster) + "/layout";
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a97d9790/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java
new file mode 100644
index 0000000..96b0771
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedLayoutFactoryImpl.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.manager.indexserver;
+
+import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS;
+
+import org.apache.blur.BlurConfiguration;
+import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
+import org.apache.zookeeper.ZooKeeper;
+
+public class DistributedLayoutFactoryImpl {
+
+  public static DistributedLayoutFactory getDistributedLayoutFactory(BlurConfiguration configuration,
String cluster,
+      ZooKeeper zooKeeper) {
+    String distributedLayoutFactoryClass = configuration.get(BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS,
"");
+    if (distributedLayoutFactoryClass.isEmpty()) {
+      String storagePath = ZookeeperPathConstants.getShardLayoutPath(cluster);
+      return new MasterBasedDistributedLayoutFactory(zooKeeper, storagePath);
+    }
+    try {
+      Class<?> clazz = Class.forName(distributedLayoutFactoryClass);
+      return (DistributedLayoutFactory) clazz.newInstance();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a97d9790/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
index b5fc5ab..e865da7 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -72,9 +73,10 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
   public DistributedLayout createDistributedLayout(String table, List<String> shardList,
List<String> shardServerList,
       List<String> offlineShardServers) {
     MasterBasedDistributedLayout layout = _cachedLayoutMap.get(table);
-    if (layout == null || layout.isOutOfDate(shardList, shardServerList, offlineShardServers))
{
+    List<String> onlineShardServerList = getOnlineShardServerList(shardServerList,
offlineShardServers);
+    if (layout == null || layout.isOutOfDate(shardList, onlineShardServerList)) {
       LOG.info("Layout out of date, recalculating for table [{0}].", table);
-      MasterBasedDistributedLayout newLayout = newLayout(table, shardList, shardServerList,
offlineShardServers);
+      MasterBasedDistributedLayout newLayout = newLayout(table, shardList, onlineShardServerList);
       _cachedLayoutMap.put(table, newLayout);
       return newLayout;
     } else {
@@ -82,8 +84,14 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
     }
   }
 
-  private MasterBasedDistributedLayout newLayout(String table, List<String> shardList,
List<String> shardServerList,
-      List<String> offlineShardServers) {
+  private List<String> getOnlineShardServerList(List<String> shardServerList,
List<String> offlineShardServers) {
+    List<String> list = new ArrayList<String>(shardServerList);
+    list.removeAll(offlineShardServers);
+    return list;
+  }
+
+  private MasterBasedDistributedLayout newLayout(String table, List<String> onlineShardServerList,
+      List<String> shardServerList) {
     try {
       _zooKeeperLockManager.lock(table);
       String storagePath = getStoragePath(table);
@@ -93,7 +101,7 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
         byte[] data = _zooKeeper.getData(storagePath, false, stat);
         if (data != null) {
           MasterBasedDistributedLayout storedLayout = fromBytes(data);
-          if (!storedLayout.isOutOfDate(shardList, shardServerList, offlineShardServers))
{
+          if (!storedLayout.isOutOfDate(onlineShardServerList, shardServerList)) {
             LOG.info("New layout fetched.");
             return storedLayout;
           }
@@ -103,9 +111,10 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
         }
       }
       // recreate
-      Map<String, String> newCalculatedLayout = calculateNewLayout(table, existingLayout,
shardList, shardServerList);
-      MasterBasedDistributedLayout layout = new MasterBasedDistributedLayout(newCalculatedLayout,
shardList,
+      Map<String, String> newCalculatedLayout = calculateNewLayout(table, existingLayout,
onlineShardServerList,
           shardServerList);
+      MasterBasedDistributedLayout layout = new MasterBasedDistributedLayout(newCalculatedLayout,
+          onlineShardServerList, shardServerList);
       if (_zooKeeper.exists(storagePath, false) == null) {
         _zooKeeper.create(storagePath, toBytes(layout), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
       } else {
@@ -127,16 +136,16 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
   }
 
   private Map<String, String> calculateNewLayout(String table, MasterBasedDistributedLayout
existingLayout,
-      List<String> shardList, List<String> shardServerList) {
-    Set<String> shardServerSet = new TreeSet<String>(shardServerList);
+      List<String> shardList, List<String> onlineShardServerList) {
+    Set<String> shardServerSet = new TreeSet<String>(onlineShardServerList);
     if (existingLayout == null) {
       // blind setup, basic round robin
       LOG.info("Blind shard layout.");
       Map<String, String> newLayoutMap = new TreeMap<String, String>();
-      Iterator<String> iterator = shardServerList.iterator();
+      Iterator<String> iterator = onlineShardServerList.iterator();
       for (String shard : shardList) {
         if (!iterator.hasNext()) {
-          iterator = shardServerList.iterator();
+          iterator = onlineShardServerList.iterator();
         }
         String server = iterator.next();
         newLayoutMap.put(shard, server);
@@ -159,7 +168,7 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
 
       LOG.info("Adding in new shard servers for table [{0}]", table);
       // Add counts for new shard servers
-      for (String server : shardServerList) {
+      for (String server : onlineShardServerList) {
         if (!onlineServerShardCount.containsKey(server)) {
           LOG.info("New shard server [{0}]", server);
           onlineServerShardCount.put(server, 0);
@@ -179,7 +188,7 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
 
       LOG.info("Leveling any shard hotspots for table [{0}]", table);
       // Level shards
-      MasterBasedLeveler.level(shardList.size(), shardServerList.size(), onlineServerShardCount,
newLayoutMap);
+      MasterBasedLeveler.level(shardList.size(), onlineShardServerList.size(), onlineServerShardCount,
newLayoutMap);
       return newLayoutMap;
     }
   }
@@ -233,13 +242,13 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
   static class MasterBasedDistributedLayout implements DistributedLayout, Serializable {
 
     private final SortedSet<String> _shardList;
-    private final SortedSet<String> _shardServerList;
+    private final SortedSet<String> _onlineShardServerList;
     private final Map<String, String> _layout;
 
     public MasterBasedDistributedLayout(Map<String, String> layout, Collection<String>
shardList,
-        Collection<String> shardServerList) {
+        Collection<String> onlineShardServerList) {
       _shardList = new TreeSet<String>(shardList);
-      _shardServerList = new TreeSet<String>(shardServerList);
+      _onlineShardServerList = new TreeSet<String>(onlineShardServerList);
       _layout = layout;
     }
 
@@ -248,8 +257,8 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac
       return _layout;
     }
 
-    public boolean isOutOfDate(List<String> shardList, List<String> shardServerList,
List<String> offlineShardServers) {
-      if (!_shardServerList.equals(new TreeSet<String>(shardServerList))) {
+    public boolean isOutOfDate(List<String> shardList, List<String> onlineShardServerList)
{
+      if (!_onlineShardServerList.equals(new TreeSet<String>(onlineShardServerList)))
{
         return true;
       } else if (!_shardList.equals(new TreeSet<String>(shardList))) {
         return true;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a97d9790/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
index b1b0bf7..1ec7c6a 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
@@ -24,10 +24,10 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.UUID;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -46,7 +46,9 @@ import org.apache.blur.manager.BlurPartitioner;
 import org.apache.blur.manager.BlurQueryChecker;
 import org.apache.blur.manager.IndexManager;
 import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
-import org.apache.blur.manager.indexserver.DistributedLayoutManager;
+import org.apache.blur.manager.indexserver.DistributedLayout;
+import org.apache.blur.manager.indexserver.DistributedLayoutFactory;
+import org.apache.blur.manager.indexserver.DistributedLayoutFactoryImpl;
 import org.apache.blur.manager.results.BlurResultIterable;
 import org.apache.blur.manager.results.BlurResultIterableClient;
 import org.apache.blur.manager.results.LazyBlurResult;
@@ -125,6 +127,7 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
   private int _remoteFetchCount = 100;
   private BlurQueryChecker _queryChecker;
   private AtomicBoolean _running = new AtomicBoolean();
+  private Map<String, DistributedLayoutFactory> _distributedLayoutFactoryMap = new
ConcurrentHashMap<String, DistributedLayoutFactory>();
 
   private int _maxFetchRetries = 3;
   private int _maxMutateRetries = 3;
@@ -167,6 +170,11 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
     _watchForClusters.watch(new OnChange() {
       @Override
       public void action(List<String> children) {
+        for (String cluster : new HashSet<String>(_distributedLayoutFactoryMap.keySet()))
{
+          if (!children.contains(cluster)) {
+            _distributedLayoutFactoryMap.remove(cluster);
+          }
+        }
         for (String cluster : children) {
           try {
             watchForLayoutChanges(cluster);
@@ -233,7 +241,6 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
     List<String> tableList = _clusterStatus.getTableList(false);
     HashMap<String, Map<String, String>> newLayout = new HashMap<String, Map<String,
String>>();
     for (String table : tableList) {
-      DistributedLayoutManager layoutManager = new DistributedLayoutManager();
       String cluster = _clusterStatus.getCluster(false, table);
       if (cluster == null) {
         continue;
@@ -241,16 +248,27 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
       List<String> shardServerList = _clusterStatus.getShardServerList(cluster);
       List<String> offlineShardServers = _clusterStatus.getOfflineShardServers(false,
cluster);
       List<String> shardList = getShardList(cluster, table);
-      layoutManager.setNodes(shardServerList);
-      layoutManager.setNodesOffline(offlineShardServers);
-      layoutManager.setShards(shardList);
-      layoutManager.init();
-      Map<String, String> layout = layoutManager.getLayout();
-      newLayout.put(table, layout);
+
+      DistributedLayoutFactory distributedLayoutFactory = getDistributedLayoutFactory(cluster);
+      DistributedLayout layout = distributedLayoutFactory.createDistributedLayout(table,
shardList, shardServerList,
+          offlineShardServers);
+      Map<String, String> map = layout.getLayout();
+      LOG.info("New layout for table [{0}] is [{1}]", table, map);
+      newLayout.put(table, map);
     }
     _shardServerLayout.set(newLayout);
   }
 
+  private synchronized DistributedLayoutFactory getDistributedLayoutFactory(String cluster)
{
+    DistributedLayoutFactory distributedLayoutFactory = _distributedLayoutFactoryMap.get(cluster);
+    if (distributedLayoutFactory == null) {
+      distributedLayoutFactory = DistributedLayoutFactoryImpl.getDistributedLayoutFactory(_configuration,
cluster,
+          _zookeeper);
+      _distributedLayoutFactoryMap.put(cluster, distributedLayoutFactory);
+    }
+    return distributedLayoutFactory;
+  }
+
   private List<String> getShardList(String cluster, String table) {
     List<String> shards = new ArrayList<String>();
     TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, cluster, table);
@@ -360,6 +378,8 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
           BlurResults results = convertToBlurResults(hitsIterable, blurQuery, facetCounts,
_executor, selector, table);
           if (!validResults(results, shardCount, blurQuery)) {
             BlurClientManager.sleep(_defaultDelay, _maxDefaultDelay, retries, _maxDefaultRetries);
+            Map<String, String> map = _shardServerLayout.get().get(table);
+            LOG.info("Current layout for table [{0}] is [{1}]", table, map);
             continue OUTER;
           }
           return results;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a97d9790/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
index 347f1ca..d3eaf80 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
@@ -130,6 +130,7 @@ public class ThriftBlurControllerServer extends ThriftServer {
     controllerServer.setMaxMutateDelay(configuration.getInt(BLUR_CONTROLLER_RETRY_MAX_MUTATE_DELAY,
2000));
     controllerServer.setMaxDefaultDelay(configuration.getInt(BLUR_CONTROLLER_RETRY_MAX_DEFAULT_DELAY,
2000));
     controllerServer.setMaxRecordsPerRowFetchRequest(configuration.getInt(BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST,
1000));
+    controllerServer.setConfiguration(configuration);
 
     controllerServer.init();
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a97d9790/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 4f6bc43..82cf541 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -36,7 +36,6 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTER_CACHE_CLASS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_HOSTNAME;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_CLASS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WARMUP_THROTTLE;
-import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INTERNAL_SEARCH_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_OPENER_THREAD_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SAFEMODEDELAY;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_SERVER_THRIFT_THREAD_COUNT;
@@ -62,7 +61,6 @@ import org.apache.blur.manager.BlurFilterCache;
 import org.apache.blur.manager.BlurQueryChecker;
 import org.apache.blur.manager.DefaultBlurFilterCache;
 import org.apache.blur.manager.IndexManager;
-import org.apache.blur.manager.clusterstatus.ClusterStatus;
 import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus;
 import org.apache.blur.manager.indexserver.BlurIndexWarmup;
 import org.apache.blur.manager.indexserver.BlurServerShutDown;
@@ -70,6 +68,7 @@ import org.apache.blur.manager.indexserver.BlurServerShutDown.BlurShutdown;
 import org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup;
 import org.apache.blur.manager.indexserver.DistributedIndexServer;
 import org.apache.blur.manager.indexserver.DistributedLayoutFactory;
+import org.apache.blur.manager.indexserver.DistributedLayoutFactoryImpl;
 import org.apache.blur.manager.writer.BlurIndexRefresher;
 import org.apache.blur.metrics.JSONReporter;
 import org.apache.blur.metrics.ReporterSetup;
@@ -213,7 +212,8 @@ public class ThriftBlurShardServer extends ThriftServer {
 
     final ZooKeeper zooKeeper = ZkUtils.newZooKeeper(zkConnectionStr, sessionTimeout);
 
-    BlurUtil.setupZookeeper(zooKeeper, configuration.get(BLUR_CLUSTER_NAME));
+    String cluster = configuration.get(BLUR_CLUSTER_NAME, BLUR_CLUSTER);
+    BlurUtil.setupZookeeper(zooKeeper, cluster);
 
     final ZookeeperClusterStatus clusterStatus = new ZookeeperClusterStatus(zooKeeper, configuration);
 
@@ -222,18 +222,18 @@ public class ThriftBlurShardServer extends ThriftServer {
 
     BlurFilterCache filterCache = getFilterCache(configuration);
     BlurIndexWarmup indexWarmup = getIndexWarmup(configuration);
-    //@todo add in read from config
-    DistributedLayoutFactory distributedLayoutFactory = null;
 
-    String cluster = configuration.get(BLUR_CLUSTER_NAME, BLUR_CLUSTER);
+    DistributedLayoutFactory distributedLayoutFactory = DistributedLayoutFactoryImpl.getDistributedLayoutFactory(
+        configuration, cluster, zooKeeper);
+
     long safeModeDelay = configuration.getLong(BLUR_SHARD_SAFEMODEDELAY, 60000);
     int shardOpenerThreadCount = configuration.getInt(BLUR_SHARD_OPENER_THREAD_COUNT, 16);
     int internalSearchThreads = configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 16);
     int warmupThreads = configuration.getInt(BLUR_SHARD_WARMUP_THREAD_COUNT, 16);
-    
+
     final DistributedIndexServer indexServer = new DistributedIndexServer(config, zooKeeper,
clusterStatus,
-        indexWarmup, filterCache, blockCacheDirectoryFactory, distributedLayoutFactory, cluster,
nodeName, safeModeDelay, shardOpenerThreadCount,
-        internalSearchThreads, warmupThreads);
+        indexWarmup, filterCache, blockCacheDirectoryFactory, distributedLayoutFactory, cluster,
nodeName,
+        safeModeDelay, shardOpenerThreadCount, internalSearchThreads, warmupThreads);
 
     final IndexManager indexManager = new IndexManager();
     indexManager.setIndexServer(indexServer);
@@ -254,6 +254,7 @@ public class ThriftBlurShardServer extends ThriftServer {
     shardServer.setQueryChecker(queryChecker);
     shardServer.setConfiguration(configuration);
     shardServer.setMaxRecordsPerRowFetchRequest(configuration.getInt(BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST,
1000));
+    shardServer.setConfiguration(configuration);
     shardServer.init();
 
     Iface iface = BlurUtil.recordMethodCallsAndAverageTimes(shardServer, Iface.class, false);
@@ -308,10 +309,10 @@ public class ThriftBlurShardServer extends ThriftServer {
   }
 
   private static BlurFilterCache getFilterCache(BlurConfiguration configuration) {
-    String _blurFilterCacheClass = configuration.get(BLUR_SHARD_FILTER_CACHE_CLASS);
-    if (_blurFilterCacheClass != null) {
+    String blurFilterCacheClass = configuration.get(BLUR_SHARD_FILTER_CACHE_CLASS);
+    if (blurFilterCacheClass != null) {
       try {
-        Class<?> clazz = Class.forName(_blurFilterCacheClass);
+        Class<?> clazz = Class.forName(blurFilterCacheClass);
         return (BlurFilterCache) clazz.newInstance();
       } catch (Exception e) {
         throw new RuntimeException(e);
@@ -321,11 +322,11 @@ public class ThriftBlurShardServer extends ThriftServer {
   }
 
   private static BlurIndexWarmup getIndexWarmup(BlurConfiguration configuration) {
-    String _blurFilterCacheClass = configuration.get(BLUR_SHARD_INDEX_WARMUP_CLASS);
-    if (_blurFilterCacheClass != null && _blurFilterCacheClass.isEmpty()) {
-      if (!_blurFilterCacheClass.equals("org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup"))
{
+    String blurFilterCacheClass = configuration.get(BLUR_SHARD_INDEX_WARMUP_CLASS);
+    if (blurFilterCacheClass != null && blurFilterCacheClass.isEmpty()) {
+      if (!blurFilterCacheClass.equals("org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup"))
{
         try {
-          Class<?> clazz = Class.forName(_blurFilterCacheClass);
+          Class<?> clazz = Class.forName(blurFilterCacheClass);
           return (BlurIndexWarmup) clazz.newInstance();
         } catch (Exception e) {
           throw new RuntimeException(e);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a97d9790/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
index 39b55bc..7fba16b 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
@@ -293,11 +293,15 @@ public class BlurClusterTest {
       if (blurException != null) {
         return blurException;
       }
-      Thread.sleep(250);
+      Thread.sleep(100);
       if (bufferToPutGcWatcherOverLimitList != null) {
-        System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() +
"] Max [" + getMaxHeapSize()
-            + "]");
-        bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]);
+        if (getHeapSize() < (getMaxHeapSize() * 0.8)) {
+          System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize()
+ "] Max ["
+              + getMaxHeapSize() + "]");
+          bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]);
+        } else {
+          System.out.println("Already allocated enough Heap [" + getHeapSize() + "] Max ["
+ getMaxHeapSize() + "]");
+        }
       }
     }
     return null;
@@ -313,6 +317,12 @@ public class BlurClusterTest {
 
   public void testTestShardFailover() throws BlurException, TException, InterruptedException,
IOException,
       KeeperException {
+    
+    System.out.println("===========================");
+    System.out.println("===========================");
+    System.out.println("===========================");
+    System.out.println("===========================");
+    
     Iface client = getClient();
     int length = 100;
     BlurQuery blurQuery = new BlurQuery();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a97d9790/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 2176ec6..1373ad2 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -75,6 +75,7 @@ public class BlurConstants {
   public static final String BLUR_SHARD_THRIFT_SELECTOR_THREADS = "blur.shard.thrift.selector.threads";
   public static final String BLUR_SHARD_THRIFT_MAX_READ_BUFFER_BYTES = "blur.shard.thrift.max.read.buffer.bytes";
   public static final String BLUR_SHARD_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.shard.thrift.accept.queue.size.per.thread";
+  public static final String BLUR_SHARD_DISTRIBUTED_LAYOUT_FACTORY_CLASS = "blur.shard.distributed.layout.factory.class";
   
   public static final String BLUR_FIELDTYPE = "blur.fieldtype.";
 


Mime
View raw message