incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] git commit: Adding more tracing to isolate where the fetch slow down occurs.
Date Fri, 29 Nov 2013 15:47:28 GMT
Updated Branches:
  refs/heads/master 54be093f3 -> 8ee31026f


Adding more tracing to isolate where the fetch slow down occurs.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/fce280d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/fce280d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/fce280d6

Branch: refs/heads/master
Commit: fce280d65364467114aefc5c01ebcc776adde29b
Parents: 54be093
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Nov 28 09:34:42 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Nov 29 10:47:26 2013 -0500

----------------------------------------------------------------------
 .../org/apache/blur/manager/IndexManager.java   |  5 +-
 .../manager/indexserver/LocalIndexServer.java   | 19 +++--
 .../java/org/apache/blur/utils/BlurUtil.java    | 10 +++
 .../apache/blur/manager/IndexManagerTest.java   | 74 +++++++++++++++++++-
 4 files changed, 102 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/fce280d6/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 25b1a8f..8804697 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -605,7 +605,10 @@ public class IndexManager {
         } else {
           fetchResult.exists = true;
           fetchResult.deleted = false;
-          String rowId = getRowId(reader, docId);
+          String rowId = selector.getRowId();
+          if (rowId == null) {
+            rowId = getRowId(reader, docId);
+          }
           Term term = new Term(ROW_ID, rowId);
           if (returnIdsOnly) {
             int recordCount = BlurUtil.countDocuments(reader, term);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/fce280d6/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
index 5e42c23..51a7468 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
@@ -52,6 +52,7 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.store.RAMDirectory;
 
 import com.google.common.io.Closer;
 
@@ -65,15 +66,20 @@ public class LocalIndexServer extends AbstractIndexServer {
   private final ExecutorService _searchExecutor;
   private final TableContext _tableContext;
   private final Closer _closer;
+  private final boolean _ramDir;
 
   public LocalIndexServer(TableDescriptor tableDescriptor) throws IOException {
+    this(tableDescriptor, false);
+  }
+
+  public LocalIndexServer(TableDescriptor tableDescriptor, boolean ramDir) throws IOException
{
     _closer = Closer.create();
     _tableContext = TableContext.create(tableDescriptor);
     _mergeScheduler = _closer.register(new SharedMergeScheduler(3));
     _gc = _closer.register(new DirectoryReferenceFileGC());
     _searchExecutor = Executors.newCachedThreadPool();
     _closer.register(new CloseableExecutorService(_searchExecutor));
-
+    _ramDir = ramDir;
     getIndexes(_tableContext.getTable());
   }
 
@@ -129,10 +135,15 @@ public class LocalIndexServer extends AbstractIndexServer {
       Map<String, BlurIndex> shards = new ConcurrentHashMap<String, BlurIndex>();
       int shardCount = _tableContext.getDescriptor().getShardCount();
       for (int i = 0; i < shardCount; i++) {
+        Directory directory;
         String shardName = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, i);
-        File file = new File(tableFile, shardName);
-        file.mkdirs();
-        MMapDirectory directory = new MMapDirectory(file);
+        if (_ramDir) {
+          directory = new RAMDirectory();
+        } else {
+          File file = new File(tableFile, shardName);
+          file.mkdirs();
+          directory = new MMapDirectory(file);
+        }
         if (!DirectoryReader.indexExists(directory)) {
           new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer())).close();
         }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/fce280d6/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index 1b69d35..4f2e830 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -759,6 +759,7 @@ public class BlurUtil {
   public static List<Document> fetchDocuments(IndexReader reader, Term term,
       ResetableDocumentStoredFieldVisitor fieldSelector, Selector selector, int maxHeap,
String context)
       throws IOException {
+    Tracer trace = Trace.trace("search for docs to fetch");
     IndexSearcher indexSearcher = new IndexSearcher(reader);
     int docFreq = reader.docFreq(term);
 
@@ -790,6 +791,10 @@ public class BlurUtil {
       topDocs = null;
     }
 
+    trace.done();
+
+    Tracer trace2 = Trace.trace("fetching doc from index");
+    long readTime = 0;
     int totalHeap = 0;
     for (int i = start; i < end; i++) {
       if (i >= totalHits) {
@@ -801,12 +806,17 @@ public class BlurUtil {
         break;
       }
       int doc = scoreDocs.get(i).doc;
+      long s = System.nanoTime();
       indexSearcher.doc(doc, fieldSelector);
+      long e = System.nanoTime();
+      readTime += (e - s);
       docs.add(fieldSelector.getDocument());
       int heapSize = fieldSelector.getSize();
       totalHeap += heapSize;
       fieldSelector.reset();
     }
+    System.out.println("Read time " + readTime);
+    trace2.done();
     return docs;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/fce280d6/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
index 48ccab3..47342e2 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
@@ -42,7 +42,9 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.TreeSet;
+import java.util.UUID;
 import java.util.concurrent.atomic.AtomicLongArray;
 
 import org.apache.blur.BlurConfiguration;
@@ -64,10 +66,14 @@ import org.apache.blur.thrift.generated.Record;
 import org.apache.blur.thrift.generated.RecordMutation;
 import org.apache.blur.thrift.generated.Row;
 import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
 import org.apache.blur.thrift.generated.Schema;
 import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.TraceCollector;
+import org.apache.blur.trace.TraceReporter;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurIterator;
 import org.apache.blur.utils.BlurUtil;
@@ -102,7 +108,7 @@ public class IndexManagerTest {
     tableDescriptor.setTableUri(file.toURI().toString());
     tableDescriptor.putToTableProperties("blur.shard.time.between.refreshs", Long.toString(100));
     tableDescriptor.setShardCount(1);
-    server = new LocalIndexServer(tableDescriptor);
+    server = new LocalIndexServer(tableDescriptor, true);
 
     BlurFilterCache filterCache = new DefaultBlurFilterCache(new BlurConfiguration());
     long statusCleanupTimerDelay = 1000;
@@ -267,6 +273,72 @@ public class IndexManagerTest {
   }
 
   @Test
+  public void testMutationReplaceLargeRow() throws Exception {
+    String rowId = "largerow";
+    indexManager.mutate(getLargeRow(rowId));
+    TraceReporter oldReporter = Trace.getReporter();
+    Trace.setReporter(new TraceReporter(new BlurConfiguration()) {
+      
+      @Override
+      public void close() throws IOException {
+        
+      }
+      
+      @Override
+      public void report(TraceCollector collector) {
+        System.out.println(collector.toJson());
+      }
+    });
+    
+    Trace.setupTrace(rowId);
+
+    Selector selector = new Selector().setRowId(rowId);
+    FetchResult fetchResult = new FetchResult();
+    long s = System.nanoTime();
+    indexManager.fetchRow(TABLE, selector, fetchResult);
+    long e = System.nanoTime();
+    assertNotNull(fetchResult.rowResult.row);
+    Trace.tearDownTrace();
+    System.out.println((e - s) / 1000000.0);
+    
+    Trace.setReporter(oldReporter);
+
+  }
+
+  private RowMutation getLargeRow(String rowId) {
+    RowMutation rowMutation = new RowMutation();
+    rowMutation.setTable(TABLE);
+    rowMutation.setRowId(rowId);
+    rowMutation.setWal(true);
+    rowMutation.setWaitToBeVisible(true);
+    rowMutation.setRecordMutations(getRecordMutations(10000));
+    rowMutation.setRowMutationType(RowMutationType.REPLACE_ROW);
+    return rowMutation;
+  }
+
+  private List<RecordMutation> getRecordMutations(int count) {
+    List<RecordMutation> mutations = new ArrayList<RecordMutation>();
+    for (int i = 0; i < count; i++) {
+      mutations.add(getRecordMutation());
+    }
+    return mutations;
+  }
+
+  private RecordMutation getRecordMutation() {
+    RecordMutation mutation = new RecordMutation();
+    mutation.setRecordMutationType(REPLACE_ENTIRE_RECORD);
+    Record record = new Record();
+    record.setFamily(FAMILY);
+    record.setRecordId(UUID.randomUUID().toString());
+    Random random = new Random();
+    for (int i = 0; i < 10; i++) {
+      record.addToColumns(new Column("col" + i, "value" + random.nextLong()));
+    }
+    mutation.setRecord(record);
+    return mutation;
+  }
+
+  @Test
   public void testFetchRowByRowIdHighlighting() throws Exception {
     Selector selector = new Selector().setRowId("row-6");
     HighlightOptions highlightOptions = new HighlightOptions();


Mime
View raw message