incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: Fixed BLUR-203
Date Mon, 12 Aug 2013 18:11:01 GMT
Fixed BLUR-203


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/71b72250
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/71b72250
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/71b72250

Branch: refs/heads/master
Commit: 71b72250c0dd93cf96522c6cba5c4bc9a431ecfc
Parents: 54bb4e2
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Aug 12 14:10:25 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Aug 12 14:10:25 2013 -0400

----------------------------------------------------------------------
 .../blur/manager/writer/BlurNRTIndex.java       |  3 +-
 .../blur/manager/writer/FieldBasedWarmer.java   | 34 +++++++++++---------
 .../apache/blur/lucene/warmup/IndexWarmup.java  | 10 ++++--
 3 files changed, 27 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/71b72250/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index 993074a..811d265 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -34,6 +34,7 @@ import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceCounter;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
 import org.apache.blur.lucene.store.refcounter.IndexInputCloser;
+import org.apache.blur.lucene.warmup.IndexWarmupIndexDeletionPolicy;
 import org.apache.blur.lucene.warmup.TraceableDirectory;
 import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.IndexSearcherClosableNRT;
@@ -88,7 +89,7 @@ public class BlurNRTIndex extends BlurIndex {
     IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, analyzer);
     conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
     conf.setSimilarity(_tableContext.getSimilarity());
-    conf.setIndexDeletionPolicy(_tableContext.getIndexDeletionPolicy());
+    conf.setIndexDeletionPolicy(new IndexWarmupIndexDeletionPolicy(_tableContext.getIndexDeletionPolicy()));
     conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, _isClosed));
 
     TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/71b72250/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
b/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
index 63d687d..822b8d4 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/FieldBasedWarmer.java
@@ -37,16 +37,17 @@ public class FieldBasedWarmer extends IndexReaderWarmer {
 
   private ShardContext shardContext;
   private AtomicBoolean isClosed;
+
   public FieldBasedWarmer(ShardContext shardContext, AtomicBoolean isClosed) {
-	this.isClosed = isClosed;
-	this.shardContext = shardContext;
+    this.isClosed = isClosed;
+    this.shardContext = shardContext;
   }
 
   @Override
   public void warm(AtomicReader reader) throws IOException {
-	ColumnPreCache columnPreCache = shardContext.getTableContext().getDescriptor().getColumnPreCache();
+    ColumnPreCache columnPreCache = shardContext.getTableContext().getDescriptor().getColumnPreCache();
     List<String> preCacheCols = columnPreCache == null ? null : columnPreCache.getPreCacheCols();
-	int maxSampleSize = 1000;
+    int maxSampleSize = 1000;
     IndexWarmup indexWarmup = new IndexWarmup(isClosed, maxSampleSize);
     String context = shardContext.getTableContext().getTable() + "/" + shardContext.getShard();
     Map<String, List<IndexTracerResult>> sampleIndex = indexWarmup.sampleIndex(reader,
context);
@@ -59,17 +60,18 @@ public class FieldBasedWarmer extends IndexReaderWarmer {
   }
 
   private void warm(AtomicReader reader, Iterable<String> preCacheCols, IndexWarmup
indexWarmup,
-	 Map<String, List<IndexTracerResult>> sampleIndex, String context, AtomicBoolean
isClosed) {
-	 for (String field : preCacheCols) {
-	   try {
-	        indexWarmup.warm(reader, sampleIndex, field, context);
-	      } catch (IOException e) {
-	        LOG.error("Context [{0}] unknown error trying to warmup the [{1}] field", e, context,
field);
-	      }
-	    if (isClosed.get()) {
-	       LOG.info("Context [{0}] index closed", context);
-	       return;
-	    }
-	}
+      Map<String, List<IndexTracerResult>> sampleIndex, String context, AtomicBoolean
isClosed) {
+    for (String field : preCacheCols) {
+      try {
+        indexWarmup.warm(reader, sampleIndex, field, context);
+      } catch (IOException e) {
+        LOG.error("Context [{0}] unknown error trying to warmup the [{1}] field", e, context,
field);
+        LOG.error("Current sampleIndex [{0}]",sampleIndex);
+      }
+      if (isClosed.get()) {
+        LOG.info("Context [{0}] index closed", context);
+        return;
+      }
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/71b72250/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java b/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
index 7dbd808..d592cde 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
@@ -155,8 +155,8 @@ public class IndexWarmup {
         double seconds = (now - start) / 1000000000.0;
         double rateMbPerSec = (bytesReadPerPass / seconds) / 1000 / 1000;
         double complete = (((double) totalLength - (double) length) / (double) totalLength)
* 100.0;
-        LOG.debug("Context [{3}] warming field [{0}] in file [{1}] is [{2}%] complete at
rate of [{4} MB/s]", fieldName,
-            fileName, complete, context, rateMbPerSec);
+        LOG.debug("Context [{3}] warming field [{0}] in file [{1}] is [{2}%] complete at
rate of [{4} MB/s]",
+            fieldName, fileName, complete, context, rateMbPerSec);
         start = System.nanoTime();
         bytesReadPerPass = 0;
         if (_isClosed.get()) {
@@ -257,7 +257,7 @@ public class IndexWarmup {
         return results;
       }
       IndexTracer tracer = new IndexTracer((TraceableDirectory) directory, _maxSampleSize);
-      String fileName = SAMPLE_PREFIX + segmentReader.getSegmentName() + SAMPLE_EXT;
+      String fileName = getSampleFileName(segmentReader.getSegmentName());
       List<IndexTracerResult> segmentTraces = new ArrayList<IndexTracerResult>();
       if (directory.fileExists(fileName)) {
         IndexInput input = directory.openInput(fileName, IOContext.READONCE);
@@ -289,6 +289,10 @@ public class IndexWarmup {
     return results;
   }
 
+  public static String getSampleFileName(String segmentName) {
+    return SAMPLE_PREFIX + segmentName + SAMPLE_EXT;
+  }
+
   private List<IndexTracerResult> read(IndexInput input) throws IOException {
     int count = input.readVInt();
     List<IndexTracerResult> results = new ArrayList<IndexTracerResult>(count);


Mime
View raw message