incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] git commit: Refactoring some code, and trying to make the blur cluster test more stable.
Date Thu, 21 Nov 2013 12:49:41 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 141b47943 -> 1b43cd906


Refactoring some code, and trying to make the blur cluster test more stable.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/dda59933
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/dda59933
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/dda59933

Branch: refs/heads/apache-blur-0.2
Commit: dda5993365eb81f474280eba4e9b4d7ef07a9039
Parents: 1266c81
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Nov 21 07:45:19 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Nov 21 07:45:19 2013 -0500

----------------------------------------------------------------------
 .../org/apache/blur/manager/IndexManager.java   | 10 ++--
 .../indexserver/DistributedIndexServer.java     | 12 +++--
 .../apache/blur/manager/writer/BlurIndex.java   | 57 ++++++++++----------
 .../blur/manager/writer/BlurIndexReadOnly.java  |  4 +-
 .../blur/manager/writer/BlurIndexReader.java    |  2 +-
 .../blur/manager/writer/BlurNRTIndex.java       |  2 +-
 .../manager/writer/TransactionRecorder.java     |  2 +-
 .../manager/writer/BlurIndexReaderTest.java     |  4 +-
 .../blur/manager/writer/BlurNRTIndexTest.java   |  8 +--
 .../org/apache/blur/thrift/BlurClusterTest.java |  2 +
 10 files changed, 55 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dda59933/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 1d833dd..0b0be0a 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -275,7 +275,7 @@ public class IndexManager {
         searcher = shardServerContext.getIndexSearcherClosable(table, shard);
       }
       if (searcher == null) {
-        searcher = index.getIndexReader();
+        searcher = index.getIndexSearcher();
         usedCache = false;
       }
 
@@ -340,7 +340,7 @@ public class IndexManager {
     if (blurIndex == null) {
       throw new BException("Shard [" + shardName + "] is not being servered by this shardserver.");
     }
-    IndexSearcherClosable searcher = blurIndex.getIndexReader();
+    IndexSearcherClosable searcher = blurIndex.getIndexSearcher();
     try {
       BooleanQuery query = new BooleanQuery();
       if (selector.recordOnly) {
@@ -692,7 +692,7 @@ public class IndexManager {
       @Override
       public Long call(Entry<String, BlurIndex> input) throws Exception {
         BlurIndex index = input.getValue();
-        IndexSearcherClosable searcher = index.getIndexReader();
+        IndexSearcherClosable searcher = index.getIndexSearcher();
         try {
           return recordFrequency(searcher.getIndexReader(), columnFamily, columnName, value);
         } finally {
@@ -728,7 +728,7 @@ public class IndexManager {
           @Override
           public List<String> call(Entry<String, BlurIndex> input) throws Exception
{
             BlurIndex index = input.getValue();
-            IndexSearcherClosable searcher = index.getIndexReader();
+            IndexSearcherClosable searcher = index.getIndexSearcher();
             try {
               return terms(searcher.getIndexReader(), columnFamily, columnName, startWith,
size);
             } finally {
@@ -1119,7 +1119,7 @@ public class IndexManager {
       String shard = entry.getKey();
       _status.attachThread(shard);
       BlurIndex index = entry.getValue();
-      IndexSearcherClosable searcher = index.getIndexReader();
+      IndexSearcherClosable searcher = index.getIndexSearcher();
       try {
         IndexReader indexReader = searcher.getIndexReader();
         if (indexReader instanceof ExitableReader) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dda59933/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index fa38de6..80b5e0d 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -46,6 +46,7 @@ import org.apache.blur.manager.clusterstatus.ClusterStatus;
 import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
 import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.manager.writer.BlurIndexCloser;
+import org.apache.blur.manager.writer.BlurIndexNRTSimple;
 import org.apache.blur.manager.writer.BlurIndexReadOnly;
 import org.apache.blur.manager.writer.BlurIndexRefresher;
 import org.apache.blur.manager.writer.BlurNRTIndex;
@@ -453,7 +454,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
     ShardContext shardContext = ShardContext.create(tableContext, shard);
 
     Directory dir;
-    
+
     TableDescriptor descriptor = tableContext.getDescriptor();
     boolean blockCacheEnabled = descriptor.isBlockCaching();
     if (blockCacheEnabled) {
@@ -464,8 +465,13 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
     }
 
     BlurIndex index;
-    
+
     BlurNRTIndex writer = new BlurNRTIndex(shardContext, _mergeScheduler, dir, _gc, _searchExecutor);
+
+    // BlurIndexNRTSimple writer = new BlurIndexNRTSimple(shardContext,
+    // _mergeScheduler, dir, _gc, _searchExecutor,
+    // _indexCloser, _refresher);
+
     if (_clusterStatus.isReadOnly(true, _cluster, table)) {
       index = new BlurIndexReadOnly(writer);
     } else {
@@ -482,7 +488,7 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
       @Override
       public void run() {
         try {
-          final IndexSearcherClosable searcher = index.getIndexReader();
+          final IndexSearcherClosable searcher = index.getIndexSearcher();
           IndexReader reader = searcher.getIndexReader();
           _warmup.warmBlurIndex(table, shard, reader, index.isClosed(), new ReleaseReader()
{
             @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dda59933/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
index 19b26b6..8100a58 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
@@ -24,8 +24,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.thrift.generated.Row;
 import org.apache.blur.utils.BlurUtil;
-import org.apache.blur.utils.ClassNameFilter;
-import org.apache.blur.utils.RamUsageEstimator;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.search.IndexSearcher;
@@ -43,7 +41,7 @@ public abstract class BlurIndex {
 
   public abstract void deleteRow(boolean waitToBeVisible, boolean wal, String rowId) throws
IOException;
 
-  public abstract IndexSearcherClosable getIndexReader() throws IOException;
+  public abstract IndexSearcherClosable getIndexSearcher() throws IOException;
 
   public abstract void close() throws IOException;
 
@@ -60,7 +58,7 @@ public abstract class BlurIndex {
   public abstract List<String> getSnapshots() throws IOException;
 
   public long getRecordCount() throws IOException {
-    IndexSearcherClosable searcher = getIndexReader();
+    IndexSearcherClosable searcher = getIndexSearcher();
     try {
       return searcher.getIndexReader().numDocs();
     } finally {
@@ -71,7 +69,7 @@ public abstract class BlurIndex {
   }
 
   public long getRowCount() throws IOException {
-    IndexSearcherClosable searcher = getIndexReader();
+    IndexSearcherClosable searcher = getIndexSearcher();
     try {
       return getRowCount(searcher);
     } finally {
@@ -87,33 +85,34 @@ public abstract class BlurIndex {
   }
 
   public long getIndexMemoryUsage() throws IOException {
-    long now = System.currentTimeMillis();
-    if (_lastMemoryCheck + ONE_MINUTE > now) {
-      return _memoryUsage;
-    }
-    IndexSearcherClosable searcher = getIndexReader();
-    try {
-      IndexReaderContext topReaderContext = searcher.getTopReaderContext();
-      return _memoryUsage = RamUsageEstimator.sizeOf(topReaderContext, new ClassNameFilter()
{
-        @Override
-        public boolean include(String className) {
-          if (className.startsWith("org.apache.blur.index.ExitableReader")) {
-            return true;
-          } else if (className.startsWith("org.apache.blur.")) {
-            // System.out.println("className [" + className + "]");
-            return false;
-          }
-          return true;
-        }
-      });
-    } finally {
-      searcher.close();
-      _lastMemoryCheck = System.currentTimeMillis();
-    }
+    return 0;
+//    long now = System.currentTimeMillis();
+//    if (_lastMemoryCheck + ONE_MINUTE > now) {
+//      return _memoryUsage;
+//    }
+//    IndexSearcherClosable searcher = getIndexReader();
+//    try {
+//      IndexReaderContext topReaderContext = searcher.getTopReaderContext();
+//      return _memoryUsage = RamUsageEstimator.sizeOf(topReaderContext, new ClassNameFilter()
{
+//        @Override
+//        public boolean include(String className) {
+//          if (className.startsWith("org.apache.blur.index.ExitableReader")) {
+//            return true;
+//          } else if (className.startsWith("org.apache.blur.")) {
+//            // System.out.println("className [" + className + "]");
+//            return false;
+//          }
+//          return true;
+//        }
+//      });
+//    } finally {
+//      searcher.close();
+//      _lastMemoryCheck = System.currentTimeMillis();
+//    }
   }
 
   public long getSegmentCount() throws IOException {
-    IndexSearcherClosable indexSearcherClosable = getIndexReader();
+    IndexSearcherClosable indexSearcherClosable = getIndexSearcher();
     try {
       IndexReader indexReader = indexSearcherClosable.getIndexReader();
       IndexReaderContext context = indexReader.getContext();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dda59933/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java
b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java
index 1092439..ddd82e1 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReadOnly.java
@@ -39,8 +39,8 @@ public class BlurIndexReadOnly extends BlurIndex {
     throw new IOException("Read Only");
   }
 
-  public IndexSearcherClosable getIndexReader() throws IOException {
-    return _blurIndex.getIndexReader();
+  public IndexSearcherClosable getIndexSearcher() throws IOException {
+    return _blurIndex.getIndexSearcher();
   }
 
   public void close() throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dda59933/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
index eb4b65e..a4653a5 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
@@ -112,7 +112,7 @@ public class BlurIndexReader extends BlurIndex {
   }
 
   @Override
-  public IndexSearcherClosable getIndexReader() throws IOException {
+  public IndexSearcherClosable getIndexSearcher() throws IOException {
     final DirectoryReader reader = _indexReaderRef.get();
     reader.incRef();
     return new IndexSearcherClosable(reader, null) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dda59933/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index f6912f2..7e77db2 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -233,7 +233,7 @@ public class BlurNRTIndex extends BlurIndex {
    * responsible for calling close on the searcher.
    */
   @Override
-  public IndexSearcherClosable getIndexReader() throws IOException {
+  public IndexSearcherClosable getIndexSearcher() throws IOException {
     return resetRunning((IndexSearcherClosable) getNRTManager().acquire());
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dda59933/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
index 05adfb5..c799376 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
@@ -400,7 +400,7 @@ public class TransactionRecorder extends TimerTask implements Closeable
{
     return fields;
   }
 
-  private Term createRowId(String id) {
+  public static Term createRowId(String id) {
     return new Term(BlurConstants.ROW_ID, id);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dda59933/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
index 5c80a9e..01762b1 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
@@ -119,12 +119,12 @@ public class BlurIndexReaderTest {
   @Test
   public void testBlurIndexWriter() throws IOException, InterruptedException, URISyntaxException
{
     setupWriter(configuration, 1);
-    IndexSearcherClosable indexReader1 = reader.getIndexReader();
+    IndexSearcherClosable indexReader1 = reader.getIndexSearcher();
     doWrite();
     assertEquals(0, indexReader1.getIndexReader().numDocs());
     indexReader1.close();
     reader.refresh();
-    IndexSearcherClosable indexReader2 = reader.getIndexReader();
+    IndexSearcherClosable indexReader2 = reader.getIndexSearcher();
     assertEquals(1, indexReader2.getIndexReader().numDocs());
     indexReader2.close();
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dda59933/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
index 9800eab..5f40fac 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurNRTIndexTest.java
@@ -129,7 +129,7 @@ public class BlurNRTIndexTest {
     int total = 0;
     for (int i = 0; i < TEST_NUMBER_WAIT_VISIBLE; i++) {
       writer.replaceRow(true, true, genRow());
-      IndexSearcherClosable searcher = writer.getIndexReader();
+      IndexSearcherClosable searcher = writer.getIndexSearcher();
       IndexReader reader = searcher.getIndexReader();
       assertEquals(i + 1, reader.numDocs());
       searcher.close();
@@ -139,7 +139,7 @@ public class BlurNRTIndexTest {
     double seconds = (e - s) / 1000000000.0;
     double rate = total / seconds;
     System.out.println("Rate " + rate);
-    IndexSearcherClosable searcher = writer.getIndexReader();
+    IndexSearcherClosable searcher = writer.getIndexSearcher();
     IndexReader reader = searcher.getIndexReader();
     assertEquals(TEST_NUMBER_WAIT_VISIBLE, reader.numDocs());
     searcher.close();
@@ -148,7 +148,7 @@ public class BlurNRTIndexTest {
   @Test
   public void testBlurIndexWriterFaster() throws IOException, InterruptedException {
     setupWriter(configuration, 100, false);
-    IndexSearcherClosable searcher1 = writer.getIndexReader();
+    IndexSearcherClosable searcher1 = writer.getIndexSearcher();
     IndexReader reader1 = searcher1.getIndexReader();
     assertEquals(0, reader1.numDocs());
     searcher1.close();
@@ -170,7 +170,7 @@ public class BlurNRTIndexTest {
     // refresh once every 25 ms
     // Thread.sleep(1000);
     writer.refresh();
-    IndexSearcherClosable searcher2 = writer.getIndexReader();
+    IndexSearcherClosable searcher2 = writer.getIndexSearcher();
     IndexReader reader2 = searcher2.getIndexReader();
     assertEquals(TEST_NUMBER, reader2.numDocs());
     searcher2.close();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dda59933/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
index ed8ae00..1dc9a03 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
@@ -370,6 +370,8 @@ public class BlurClusterTest {
               + getMaxHeapSize() + "]");
           bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]);
         } else {
+          System.gc();
+          System.gc();
           System.out.println("Already allocated enough Heap [" + getHeapSize() + "] Max ["
+ getMaxHeapSize() + "]");
         }
       }


Mime
View raw message