incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] git commit: Passing table and shard into cache directory.
Date Mon, 21 Oct 2013 01:43:10 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 f2682d5b7 -> 8f6134c73


Passing table and shard into cache directory.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/09a7bc6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/09a7bc6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/09a7bc6d

Branch: refs/heads/apache-blur-0.2
Commit: 09a7bc6d2a649b92f37224f2f920fd812787cb5a
Parents: f2682d5
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sun Oct 20 21:27:56 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sun Oct 20 21:27:56 2013 -0400

----------------------------------------------------------------------
 .../indexserver/DistributedIndexServer.java     | 24 ++++++++++----------
 .../blur/store/BlockCacheDirectoryFactory.java  |  4 ++--
 .../store/BlockCacheDirectoryFactoryV1.java     |  5 ++--
 .../store/BlockCacheDirectoryFactoryV2.java     |  5 ++--
 .../store/blockcache_v2/CacheDirectory.java     |  6 ++---
 .../blur/store/CacheDirectoryTestSuite.java     | 12 +++++-----
 .../store/blockcache_v2/CacheDirectoryTest.java |  4 ++--
 7 files changed, 31 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index ca826d1..d2f686f 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -120,8 +120,8 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
   public DistributedIndexServer(Configuration configuration, ZooKeeper zookeeper, ClusterStatus
clusterStatus,
       BlurIndexWarmup warmup, BlurFilterCache filterCache, BlockCacheDirectoryFactory blockCacheDirectoryFactory,
       DistributedLayoutFactory distributedLayoutFactory, String cluster, String nodeName,
long safeModeDelay,
-      int shardOpenerThreadCount, int internalSearchThreads, int warmupThreads, int maxMergeThreads)
throws KeeperException,
-      InterruptedException {
+      int shardOpenerThreadCount, int internalSearchThreads, int warmupThreads, int maxMergeThreads)
+      throws KeeperException, InterruptedException {
     super(clusterStatus, configuration, nodeName, cluster);
     _closer = Closer.create();
     _shardOpenerThreadCount = shardOpenerThreadCount;
@@ -133,23 +133,23 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
     _warmupThreads = warmupThreads;
     _blockCacheDirectoryFactory = blockCacheDirectoryFactory;
     _distributedLayoutFactory = distributedLayoutFactory == null ? getDefaultLayoutFactory()
: distributedLayoutFactory;
-    
+
     _closer.register(_shardStateManager);
 
     BlurUtil.setupZookeeper(_zookeeper, _cluster);
     _openerService = Executors.newThreadPool("shard-opener", _shardOpenerThreadCount);
     _searchExecutor = Executors.newThreadPool("internal-search", _internalSearchThreads);
     _warmupExecutor = Executors.newThreadPool("warmup", _warmupThreads);
-    
+
     _closer.register(CloseableExecutorService.close(_openerService));
     _closer.register(CloseableExecutorService.close(_searchExecutor));
     _closer.register(CloseableExecutorService.close(_warmupExecutor));
-    
+
     _gc = _closer.register(new DirectoryReferenceFileGC());
 
     // @TODO allow for configuration of these
     _mergeScheduler = _closer.register(new SharedMergeScheduler(maxMergeThreads));
-    
+
     _refresher = _closer.register(new BlurIndexRefresher());
     _indexCloser = _closer.register(new BlurIndexCloser());
     _timerCacheFlush = setupFlushCacheTimer();
@@ -158,9 +158,9 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
     String onlineShardsPath = ZookeeperPathConstants.getOnlineShardsPath(_cluster);
     String safemodePath = ZookeeperPathConstants.getSafemodePath(_cluster);
 
-    //Set the registerNode timeout value to zk sessionTimeout + {4} seconds
-    int registerNodeTimeOut = _zookeeper.getSessionTimeout()/1000 + 4;
-      
+    // Set the registerNode timeout value to zk sessionTimeout + {4} seconds
+    int registerNodeTimeOut = _zookeeper.getSessionTimeout() / 1000 + 4;
+
     SafeMode safeMode = new SafeMode(_zookeeper, safemodePath, onlineShardsPath, TimeUnit.MILLISECONDS,
_safeModeDelay,
         TimeUnit.SECONDS, registerNodeTimeOut);
     safeMode.registerNode(getNodeName(), BlurUtil.getVersion().getBytes());
@@ -169,7 +169,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
     _timerTableWarmer = setupTableWarmer();
     _watchOnlineShards = watchForShardServerChanges();
   }
-  
+
   @Override
   public void close() throws IOException {
     if (_running.get()) {
@@ -456,7 +456,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
     boolean blockCacheEnabled = _clusterStatus.isBlockCacheEnabled(_cluster, table);
     if (blockCacheEnabled) {
       Set<String> blockCacheFileTypes = _clusterStatus.getBlockCacheFileTypes(_cluster,
table);
-      dir = _blockCacheDirectoryFactory.newDirectory(table + "_" + shard, directory, blockCacheFileTypes);
+      dir = _blockCacheDirectoryFactory.newDirectory(table, shard, directory, blockCacheFileTypes);
     } else {
       dir = directory;
     }
@@ -575,7 +575,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
 
     DistributedLayout layoutManager = _distributedLayoutFactory.createDistributedLayout(table,
shardList,
         shardServerList, offlineShardServers);
-    
+
     Map<String, String> layout = layoutManager.getLayout();
     String nodeName = getNodeName();
     Set<String> shardsToServeCache = new TreeSet<String>();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactory.java
b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactory.java
index 4b10ced..a02d4cb 100644
--- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactory.java
+++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactory.java
@@ -23,7 +23,7 @@ import org.apache.lucene.store.Directory;
 
 public abstract class BlockCacheDirectoryFactory {
 
-  public abstract Directory newDirectory(String name, Directory directory, Set<String>
blockCacheFileTypes)
-      throws IOException;
+  public abstract Directory newDirectory(String table, String shard, Directory directory,
+      Set<String> blockCacheFileTypes) throws IOException;
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java
b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java
index e4ad073..254b765 100644
--- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java
+++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV1.java
@@ -75,8 +75,9 @@ public class BlockCacheDirectoryFactoryV1 extends BlockCacheDirectoryFactory
{
   }
 
   @Override
-  public Directory newDirectory(String name, Directory directory, Set<String> blockCacheFileTypes)
throws IOException {
-    return new BlockDirectory(name, directory, _cache, blockCacheFileTypes);
+  public Directory newDirectory(String table, String shard, Directory directory, Set<String>
blockCacheFileTypes)
+      throws IOException {
+    return new BlockDirectory(table + "_" + shard, directory, _cache, blockCacheFileTypes);
   }
 
   private static int getSlabCount(int slabCount, int numberOfBlocksPerSlab, int blockSize,
long totalNumberOfBytes) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
index 6d1a47e..aff5823 100644
--- a/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
+++ b/blur-store/src/main/java/org/apache/blur/store/BlockCacheDirectoryFactoryV2.java
@@ -90,8 +90,9 @@ public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory
{
   }
 
   @Override
-  public Directory newDirectory(String name, Directory directory, Set<String> blockCacheFileTypes)
throws IOException {
-    return new CacheDirectory(name, directory, _cache);
+  public Directory newDirectory(String table, String shard, Directory directory, Set<String>
blockCacheFileTypes)
+      throws IOException {
+    return new CacheDirectory(table, shard, directory, _cache);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java
index 73cc147..341c409 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheDirectory.java
@@ -35,12 +35,12 @@ public class CacheDirectory extends Directory implements DirectoryDecorator,
Las
   private final String _directoryName;
   private final Cache _cache;
 
-  public CacheDirectory(String directoryName, Directory directory, Cache cache) {
+  public CacheDirectory(String table, String shard, Directory directory, Cache cache) {
     if (!(directory instanceof LastModified)) {
       throw new RuntimeException("Directory [" + directory + "] does not implement '" + LastModified.class.toString()
           + "'");
     }
-    _directoryName = notNull(directoryName);
+    _directoryName = notNull(table + "_" + shard);
     _internal = notNull(directory);
     _cache = notNull(cache);
   }
@@ -136,7 +136,7 @@ public class CacheDirectory extends Directory implements DirectoryDecorator,
Las
   public Directory getOriginalDirectory() {
     return _internal;
   }
-  
+
   private static <T> T notNull(T t) {
     if (t == null) {
       throw new IllegalArgumentException("Cannot be null");

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
index b03053e..38e448f 100644
--- a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
+++ b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
@@ -38,21 +38,21 @@ public abstract class CacheDirectoryTestSuite extends BaseDirectoryTestSuite
{
     int totalNumberOfBytes = 1000000;
     final int fileBufferSizeInt = numberBetween(113, 215);
     final int cacheBlockSizeInt = numberBetween(111, 251);
-    
+
     Size fileBufferSize = new Size() {
       @Override
       public int getSize(String directoryName, String fileName) {
         return fileBufferSizeInt;
       }
     };
-    
+
     Size cacheBlockSize = new Size() {
       @Override
       public int getSize(String directoryName, String fileName) {
         return cacheBlockSizeInt;
       }
     };
-    
+
     FileNameFilter writeFilter = new FileNameFilter() {
       @Override
       public boolean accept(String directoryName, String fileName) {
@@ -71,16 +71,16 @@ public abstract class CacheDirectoryTestSuite extends BaseDirectoryTestSuite
{
         return false;
       }
     };
-    _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter,quiet,
+    _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter,
writeFilter, quiet,
         getStore());
     Directory dir = FSDirectory.open(new File(file, "cache"));
 
     BufferStore.init(128, 128);
-    return new CacheDirectory("test", wrapLastModified(dir), _cache);
+    return new CacheDirectory("test", "test", wrapLastModified(dir), _cache);
   }
 
   protected abstract STORE getStore();
-  
+
   public void close() throws IOException {
     _cache.close();
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/09a7bc6d/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
index 8867cff..b351fcf 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheDirectoryTest.java
@@ -91,9 +91,9 @@ public class CacheDirectoryTest {
         STORE.ON_HEAP);
     Directory directory = newDirectory();
     BufferStore.init(128, 128);
-    _cacheDirectory = new CacheDirectory("test", directory, _cache);
+    _cacheDirectory = new CacheDirectory("test", "test", directory, _cache);
   }
-  
+
   @After
   public void tearDown() throws IOException {
     _cache.close();


Mime
View raw message