incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject git commit: BLUR:148 warm up new segments after NRT reload
Date Tue, 25 Jun 2013 04:17:03 GMT
Updated Branches:
  refs/heads/master d54a7fc81 -> 5373a5577


BLUR:148 warm up new segments after NRT reload


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/5373a557
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/5373a557
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/5373a557

Branch: refs/heads/master
Commit: 5373a55775e4e2a3b9c7ccfff88a345300646c5b
Parents: d54a7fc
Author: Gagan <gagandeepjuneja@gmail.com>
Authored: Tue Jun 25 09:46:37 2013 +0530
Committer: Gagan <gagandeepjuneja@gmail.com>
Committed: Tue Jun 25 09:46:37 2013 +0530

----------------------------------------------------------------------
 .../blur/manager/writer/BlurNRTIndex.java       |  2 +-
 .../blur/manager/writer/FieldBasedWarmer.java   | 79 ++++++++------------
 2 files changed, 31 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5373a557/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index 2b350fa..4b0bb20 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -85,7 +85,7 @@ public class BlurNRTIndex extends BlurIndex {
     conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
     conf.setSimilarity(_tableContext.getSimilarity());
     conf.setIndexDeletionPolicy(_tableContext.getIndexDeletionPolicy());
-    conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext));
+    conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, _isClosed));
 
     TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
     mergePolicy.setUseCompoundFile(false);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5373a557/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
b/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
index 7b275d2..63d687d 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
@@ -18,13 +18,15 @@ package org.apache.blur.manager.writer;
  */
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
+import org.apache.blur.lucene.warmup.IndexTracerResult;
+import org.apache.blur.lucene.warmup.IndexWarmup;
 import org.apache.blur.server.ShardContext;
-import org.apache.blur.server.TableContext;
 import org.apache.blur.thrift.generated.ColumnPreCache;
-import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
@@ -33,62 +35,41 @@ public class FieldBasedWarmer extends IndexReaderWarmer {
 
   private static final Log LOG = LogFactory.getLog(FieldBasedWarmer.class);
 
-  private List<String> preCacheCols;
-
-  public FieldBasedWarmer(ShardContext shardContext) {
-    this(shardContext.getTableContext());
-  }
-
-  public FieldBasedWarmer(TableContext tableContext) {
-    this(tableContext.getDescriptor());
-  }
-
-  public FieldBasedWarmer(TableDescriptor tableDescriptor) {
-    this(tableDescriptor.getColumnPreCache());
-  }
-
-  public FieldBasedWarmer(ColumnPreCache columnPreCache) {
-    this(columnPreCache == null ? null : columnPreCache.getPreCacheCols());
-  }
-
-  public FieldBasedWarmer(List<String> preCacheCols) {
-    this.preCacheCols = preCacheCols;
+  private ShardContext shardContext;
+  private AtomicBoolean isClosed;
+  public FieldBasedWarmer(ShardContext shardContext, AtomicBoolean isClosed) {
+	this.isClosed = isClosed;
+	this.shardContext = shardContext;
   }
 
   @Override
   public void warm(AtomicReader reader) throws IOException {
+	ColumnPreCache columnPreCache = shardContext.getTableContext().getDescriptor().getColumnPreCache();
+    List<String> preCacheCols = columnPreCache == null ? null : columnPreCache.getPreCacheCols();
+	int maxSampleSize = 1000;
+    IndexWarmup indexWarmup = new IndexWarmup(isClosed, maxSampleSize);
+    String context = shardContext.getTableContext().getTable() + "/" + shardContext.getShard();
+    Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(reader,
context);
     if (preCacheCols != null) {
-      warm(reader, preCacheCols);
+      warm(reader, columnPreCache.preCacheCols, indexWarmup, sampleIndex, context, isClosed);
     } else {
       Fields fields = reader.fields();
-      warm(reader, fields);
+      warm(reader, fields, indexWarmup, sampleIndex, context, isClosed);
     }
   }
 
-  private void warm(AtomicReader reader, Iterable<String> fieldNames) throws IOException
{
-    // for (String field : fieldNames) {
-    // LOG.debug("Warming field [{0}] in reader [{1}]", field, reader);
-    //
-    // AtomicReaderContext context = reader.getContext();
-    // context.
-    //
-    // Fields fields = reader.fields();
-    // Terms terms = fields.terms(field);
-    // TermsEnum termsEnum = terms.iterator(null);
-    // BytesRef ref = null;
-    // Term term = new Term(field);
-    // while ((ref = termsEnum.next()) != null) {
-    // term.set(field, ref);
-    // DocsAndPositionsEnum termPositionsEnum = reader.termPositionsEnum(term);
-    // if (termPositionsEnum != null) {
-    // while (termPositionsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
-    // int freq = termPositionsEnum.freq();
-    // for (int i = 0; i < freq; i++) {
-    // termPositionsEnum.nextPosition();
-    // }
-    // }
-    // }
-    // }
-    // }
+  private void warm(AtomicReader reader, Iterable<String> preCacheCols, IndexWarmup
indexWarmup,
+	 Map<String, List<IndexTracerResult>> sampleIndex, String context, AtomicBoolean
isClosed) {
+	 for (String field : preCacheCols) {
+	   try {
+	        indexWarmup.warm(reader, sampleIndex, field, context);
+	      } catch (IOException e) {
+	        LOG.error("Context [{0}] unknown error trying to warmup the [{1}] field", e, context,
field);
+	      }
+	    if (isClosed.get()) {
+	       LOG.info("Context [{0}] index closed", context);
+	       return;
+	    }
+	}
   }
 }


Mime
View raw message