incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Fixes in the CacheDecompression code and I moved the metrics collectors to correct locations.
Date Fri, 17 Jan 2014 13:27:51 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 76c55cc01 -> 9340af68a


Fixes in the CacheDecompression code and I moved the metrics collectors to correct locations.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/9340af68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/9340af68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/9340af68

Branch: refs/heads/apache-blur-0.2
Commit: 9340af68aa11dcc1123397e009dedcf3988090e6
Parents: 76c55cc
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Fri Jan 17 08:27:48 2014 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Jan 17 08:27:48 2014 -0500

----------------------------------------------------------------------
 .../org/apache/blur/manager/IndexManager.java   | 215 +++++++++----------
 .../blur/manager/writer/MutatableAction.java    |  26 ++-
 .../lucene/codec/Blur022StoredFieldsFormat.java |  23 +-
 .../blur/lucene/codec/CachedDecompressor.java   |  72 +++----
 .../apache/blur/lucene/warmup/IndexWarmup.java  |  20 +-
 5 files changed, 179 insertions(+), 177 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9340af68/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 4ca8cf1..e70d095 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -134,10 +134,16 @@ public class IndexManager {
   public static final String NOT_FOUND = "NOT_FOUND";
   private static final Log LOG = LogFactory.getLog(IndexManager.class);
 
-  private final Meter _readRecordsMeter;
-  private final Meter _readRowMeter;
-  private final Meter _writeRecordsMeter;
-  private final Meter _writeRowMeter;
+  private static final Meter _readRecordsMeter;
+  private static final Meter _readRowMeter;
+
+  static {
+    MetricName metricName1 = new MetricName(ORG_APACHE_BLUR, BLUR, "Read Records/s");
+    MetricName metricName2 = new MetricName(ORG_APACHE_BLUR, BLUR, "Read Row/s");
+    _readRecordsMeter = Metrics.newMeter(metricName1, "Records/s", TimeUnit.SECONDS);
+    _readRowMeter = Metrics.newMeter(metricName2, "Row/s", TimeUnit.SECONDS);
+  }
+
   private final Meter _queriesExternalMeter;
   private final Meter _queriesInternalMeter;
 
@@ -169,21 +175,13 @@ public class IndexManager {
     _clusterStatus = clusterStatus;
     _filterCache = filterCache;
 
-    MetricName metricName1 = new MetricName(ORG_APACHE_BLUR, BLUR, "Read Records/s");
-    MetricName metricName2 = new MetricName(ORG_APACHE_BLUR, BLUR, "Read Row/s");
-    MetricName metricName3 = new MetricName(ORG_APACHE_BLUR, BLUR, "Write Records/s");
-    MetricName metricName4 = new MetricName(ORG_APACHE_BLUR, BLUR, "Write Row/s");
-    MetricName metricName5 = new MetricName(ORG_APACHE_BLUR, BLUR, "External Queries/s");
-    MetricName metricName6 = new MetricName(ORG_APACHE_BLUR, BLUR, "Internal Queries/s");
-    MetricName metricName7 = new MetricName(ORG_APACHE_BLUR, BLUR, "Fetch Timer");
+    MetricName metricName1 = new MetricName(ORG_APACHE_BLUR, BLUR, "External Queries/s");
+    MetricName metricName2 = new MetricName(ORG_APACHE_BLUR, BLUR, "Internal Queries/s");
+    MetricName metricName3 = new MetricName(ORG_APACHE_BLUR, BLUR, "Fetch Timer");
 
-    _readRecordsMeter = Metrics.newMeter(metricName1, "Records/s", TimeUnit.SECONDS);
-    _readRowMeter = Metrics.newMeter(metricName2, "Row/s", TimeUnit.SECONDS);
-    _writeRecordsMeter = Metrics.newMeter(metricName3, "Records/s", TimeUnit.SECONDS);
-    _writeRowMeter = Metrics.newMeter(metricName4, "Row/s", TimeUnit.SECONDS);
-    _queriesExternalMeter = Metrics.newMeter(metricName5, "External Queries/s", TimeUnit.SECONDS);
-    _queriesInternalMeter = Metrics.newMeter(metricName6, "Internal Queries/s", TimeUnit.SECONDS);
-    _fetchTimer = Metrics.newTimer(metricName7, TimeUnit.MICROSECONDS, TimeUnit.SECONDS);
+    _queriesExternalMeter = Metrics.newMeter(metricName1, "External Queries/s", TimeUnit.SECONDS);
+    _queriesInternalMeter = Metrics.newMeter(metricName2, "Internal Queries/s", TimeUnit.SECONDS);
+    _fetchTimer = Metrics.newTimer(metricName3, TimeUnit.MICROSECONDS, TimeUnit.SECONDS);
 
     if (threadCount == 0) {
       throw new RuntimeException("Thread Count cannot be 0.");
@@ -308,15 +306,6 @@ public class IndexManager {
 
       fetchRow(searcher.getIndexReader(), table, shard, selector, fetchResult, highlightQuery,
fieldManager,
           _maxHeapPerRowFetch, tableContext, filter);
-
-      if (fetchResult.rowResult != null) {
-        if (fetchResult.rowResult.row != null && fetchResult.rowResult.row.records
!= null) {
-          _readRecordsMeter.mark(fetchResult.rowResult.row.records.size());
-        }
-        _readRowMeter.mark();
-      } else if (fetchResult.recordResult != null) {
-        _readRecordsMeter.mark();
-      }
     } catch (Exception e) {
       LOG.error("Unknown error while trying to fetch row.", e);
       throw new BException(e.getMessage(), e);
@@ -621,98 +610,109 @@ public class IndexManager {
   public static void fetchRow(IndexReader reader, String table, String shard, Selector selector,
       FetchResult fetchResult, Query highlightQuery, FieldManager fieldManager, int maxHeap,
TableContext tableContext,
       Filter filter) throws CorruptIndexException, IOException {
-    fetchResult.table = table;
-    String locationId = selector.locationId;
-    int lastSlash = locationId.lastIndexOf('/');
-    int docId = Integer.parseInt(locationId.substring(lastSlash + 1));
-    if (docId >= reader.maxDoc()) {
-      throw new RuntimeException("Location id [" + locationId + "] with docId [" + docId
+ "] is not valid.");
-    }
-
-    boolean returnIdsOnly = false;
-    if (selector.columnFamiliesToFetch != null && selector.columnsToFetch != null
-        && selector.columnFamiliesToFetch.isEmpty() && selector.columnsToFetch.isEmpty())
{
-      // exit early
-      returnIdsOnly = true;
-    }
+    try {
+      fetchResult.table = table;
+      String locationId = selector.locationId;
+      int lastSlash = locationId.lastIndexOf('/');
+      int docId = Integer.parseInt(locationId.substring(lastSlash + 1));
+      if (docId >= reader.maxDoc()) {
+        throw new RuntimeException("Location id [" + locationId + "] with docId [" + docId
+ "] is not valid.");
+      }
 
-    Tracer t1 = Trace.trace("fetchRow - live docs");
-    Bits liveDocs = MultiFields.getLiveDocs(reader);
-    t1.done();
-    ResetableDocumentStoredFieldVisitor fieldVisitor = getFieldSelector(selector);
-    if (selector.isRecordOnly()) {
-      // select only the row for the given data or location id.
-      if (isFiltered(docId, reader, filter)) {
-        fetchResult.exists = false;
-        fetchResult.deleted = false;
-        return;
-      } else if (liveDocs != null && !liveDocs.get(docId)) {
-        fetchResult.exists = false;
-        fetchResult.deleted = true;
-        return;
-      } else {
-        fetchResult.exists = true;
-        fetchResult.deleted = false;
-        reader.document(docId, fieldVisitor);
-        Document document = fieldVisitor.getDocument();
-        if (highlightQuery != null && fieldManager != null) {
-          HighlightOptions highlightOptions = selector.getHighlightOptions();
-          String preTag = highlightOptions.getPreTag();
-          String postTag = highlightOptions.getPostTag();
-          try {
-            document = HighlightHelper
-                .highlight(docId, document, highlightQuery, fieldManager, reader, preTag,
postTag);
-          } catch (InvalidTokenOffsetsException e) {
-            LOG.error("Unknown error while tring to highlight", e);
-          }
-        }
-        fieldVisitor.reset();
-        fetchResult.recordResult = getRecord(document);
-        return;
+      boolean returnIdsOnly = false;
+      if (selector.columnFamiliesToFetch != null && selector.columnsToFetch != null
+          && selector.columnFamiliesToFetch.isEmpty() && selector.columnsToFetch.isEmpty())
{
+        // exit early
+        returnIdsOnly = true;
       }
-    } else {
-      Tracer trace = Trace.trace("fetchRow - Row read");
-      try {
-        if (liveDocs != null && !liveDocs.get(docId)) {
+
+      Tracer t1 = Trace.trace("fetchRow - live docs");
+      Bits liveDocs = MultiFields.getLiveDocs(reader);
+      t1.done();
+      ResetableDocumentStoredFieldVisitor fieldVisitor = getFieldSelector(selector);
+      if (selector.isRecordOnly()) {
+        // select only the row for the given data or location id.
+        if (isFiltered(docId, reader, filter)) {
+          fetchResult.exists = false;
+          fetchResult.deleted = false;
+          return;
+        } else if (liveDocs != null && !liveDocs.get(docId)) {
           fetchResult.exists = false;
           fetchResult.deleted = true;
           return;
         } else {
           fetchResult.exists = true;
           fetchResult.deleted = false;
-          if (returnIdsOnly) {
-            String rowId = selector.getRowId();
-            if (rowId == null) {
-              rowId = getRowId(reader, docId);
+          reader.document(docId, fieldVisitor);
+          Document document = fieldVisitor.getDocument();
+          if (highlightQuery != null && fieldManager != null) {
+            HighlightOptions highlightOptions = selector.getHighlightOptions();
+            String preTag = highlightOptions.getPreTag();
+            String postTag = highlightOptions.getPostTag();
+            try {
+              document = HighlightHelper.highlight(docId, document, highlightQuery, fieldManager,
reader, preTag,
+                  postTag);
+            } catch (InvalidTokenOffsetsException e) {
+              LOG.error("Unknown error while tring to highlight", e);
             }
-            fetchResult.rowResult = new FetchRowResult();
-            fetchResult.rowResult.row = new Row(rowId, null);
+          }
+          fieldVisitor.reset();
+          fetchResult.recordResult = getRecord(document);
+          return;
+        }
+      } else {
+        Tracer trace = Trace.trace("fetchRow - Row read");
+        try {
+          if (liveDocs != null && !liveDocs.get(docId)) {
+            fetchResult.exists = false;
+            fetchResult.deleted = true;
+            return;
           } else {
-            List<Document> docs;
-            AtomicBoolean moreDocsToFetch = new AtomicBoolean(false);
-            AtomicInteger totalRecords = new AtomicInteger();
-            BlurHighlighter highlighter = new BlurHighlighter(highlightQuery, fieldManager,
selector);
-            Tracer docTrace = Trace.trace("fetchRow - Document read");
-            docs = BlurUtil.fetchDocuments(reader, fieldVisitor, selector, maxHeap, table
+ "/" + shard,
-                tableContext.getDefaultPrimeDocTerm(), filter, moreDocsToFetch, totalRecords,
highlighter);
-            docTrace.done();
-            Tracer rowTrace = Trace.trace("fetchRow - Row create");
-            Row row = getRow(docs);
-            if (row == null) {
+            fetchResult.exists = true;
+            fetchResult.deleted = false;
+            if (returnIdsOnly) {
               String rowId = selector.getRowId();
               if (rowId == null) {
                 rowId = getRowId(reader, docId);
               }
-              row = new Row(rowId, null);
+              fetchResult.rowResult = new FetchRowResult();
+              fetchResult.rowResult.row = new Row(rowId, null);
+            } else {
+              List<Document> docs;
+              AtomicBoolean moreDocsToFetch = new AtomicBoolean(false);
+              AtomicInteger totalRecords = new AtomicInteger();
+              BlurHighlighter highlighter = new BlurHighlighter(highlightQuery, fieldManager,
selector);
+              Tracer docTrace = Trace.trace("fetchRow - Document read");
+              docs = BlurUtil.fetchDocuments(reader, fieldVisitor, selector, maxHeap, table
+ "/" + shard,
+                  tableContext.getDefaultPrimeDocTerm(), filter, moreDocsToFetch, totalRecords,
highlighter);
+              docTrace.done();
+              Tracer rowTrace = Trace.trace("fetchRow - Row create");
+              Row row = getRow(docs);
+              if (row == null) {
+                String rowId = selector.getRowId();
+                if (rowId == null) {
+                  rowId = getRowId(reader, docId);
+                }
+                row = new Row(rowId, null);
+              }
+              fetchResult.rowResult = new FetchRowResult(row, selector.getStartRecord(),
+                  selector.getMaxRecordsToFetch(), moreDocsToFetch.get(), totalRecords.get());
+              rowTrace.done();
             }
-            fetchResult.rowResult = new FetchRowResult(row, selector.getStartRecord(), selector.getMaxRecordsToFetch(),
-                moreDocsToFetch.get(), totalRecords.get());
-            rowTrace.done();
+            return;
           }
-          return;
+        } finally {
+          trace.done();
         }
-      } finally {
-        trace.done();
+      }
+    } finally {
+      if (fetchResult.rowResult != null) {
+        if (fetchResult.rowResult.row != null && fetchResult.rowResult.row.records
!= null) {
+          _readRecordsMeter.mark(fetchResult.rowResult.row.records.size());
+        }
+        _readRowMeter.mark();
+      } else if (fetchResult.recordResult != null) {
+        _readRecordsMeter.mark();
       }
     }
   }
@@ -1106,7 +1106,7 @@ public class IndexManager {
         switch (type) {
         case REPLACE_ROW:
           Row row = MutationHelper.getRowFromMutations(mutation.rowId, mutation.recordMutations);
-          mutatableAction.replaceRow(updateMetrics(row));
+          mutatableAction.replaceRow(row);
           break;
         case UPDATE_ROW:
           doUpdateRowMutation(mutation, mutatableAction);
@@ -1146,15 +1146,6 @@ public class IndexManager {
     return map;
   }
 
-  private Row updateMetrics(Row row) {
-    _writeRowMeter.mark();
-    List<Record> records = row.getRecords();
-    if (records != null) {
-      _writeRecordsMeter.mark(records.size());
-    }
-    return row;
-  }
-
   private void doUpdateRowMutation(RowMutation mutation, MutatableAction mutatableAction)
throws BlurException,
       IOException {
     String rowId = mutation.getRowId();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9340af68/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java b/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java
index 05e99af..6d70c86 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java
@@ -16,6 +16,9 @@
  */
 package org.apache.blur.manager.writer;
 
+import static org.apache.blur.metrics.MetricsConstants.BLUR;
+import static org.apache.blur.metrics.MetricsConstants.ORG_APACHE_BLUR;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -23,6 +26,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.manager.IndexManager;
@@ -41,8 +45,22 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.Term;
 
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Meter;
+import com.yammer.metrics.core.MetricName;
+
 public class MutatableAction {
 
+  private static final Meter _writeRecordsMeter;
+  private static final Meter _writeRowMeter;
+
+  static {
+    MetricName metricName1 = new MetricName(ORG_APACHE_BLUR, BLUR, "Write Records/s");
+    MetricName metricName2 = new MetricName(ORG_APACHE_BLUR, BLUR, "Write Row/s");
+    _writeRecordsMeter = Metrics.newMeter(metricName1, "Records/s", TimeUnit.SECONDS);
+    _writeRowMeter = Metrics.newMeter(metricName2, "Row/s", TimeUnit.SECONDS);
+  }
+
   static class UpdateRow extends InternalAction {
 
     static abstract class UpdateRowAction {
@@ -224,7 +242,8 @@ public class MutatableAction {
       Row row = null;
       if (!selector.getLocationId().equals(IndexManager.NOT_FOUND)) {
         FetchResult fetchResult = new FetchResult();
-        IndexManager.fetchRow(searcher.getIndexReader(), _table, _shard, selector, fetchResult,
null, null, _maxHeap, _tableContext, null);
+        IndexManager.fetchRow(searcher.getIndexReader(), _table, _shard, selector, fetchResult,
null, null, _maxHeap,
+            _tableContext, null);
         FetchRowResult rowResult = fetchResult.getRowResult();
         if (rowResult != null) {
           row = rowResult.getRow();
@@ -237,9 +256,11 @@ public class MutatableAction {
       if (row != null && row.getRecords() != null && row.getRecords().size()
> 0) {
         List<List<Field>> docsToUpdate = RowDocumentUtil.getDocs(row, _fieldManager);
         writer.updateDocuments(term, docsToUpdate);
+        _writeRecordsMeter.mark(docsToUpdate.size());
       } else {
         writer.deleteDocuments(term);
       }
+      _writeRowMeter.mark();
     }
 
   }
@@ -268,6 +289,7 @@ public class MutatableAction {
       @Override
       void performAction(IndexSearcherClosable searcher, IndexWriter writer) throws IOException
{
         writer.deleteDocuments(createRowId(rowId));
+        _writeRowMeter.mark();
       }
     });
   }
@@ -279,6 +301,8 @@ public class MutatableAction {
         List<List<Field>> docs = RowDocumentUtil.getDocs(row, _fieldManager);
         Term rowId = createRowId(row.getId());
         writer.updateDocuments(rowId, docs);
+        _writeRecordsMeter.mark(docs.size());
+        _writeRowMeter.mark();
       }
     });
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9340af68/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java
b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java
index 04b2526..d286e6e 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java
@@ -34,10 +34,6 @@ import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.BytesRef;
-
-import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
-import com.googlecode.concurrentlinkedhashmap.Weigher;
 
 public final class Blur022StoredFieldsFormat extends StoredFieldsFormat {
 
@@ -47,31 +43,18 @@ public final class Blur022StoredFieldsFormat extends StoredFieldsFormat
{
   private static final String SEGMENT_SUFFIX = "";
   private final int _chunkSize;
   private final CompressionMode _compressionMode;
-  private final ConcurrentLinkedHashMap<CachedKey, BytesRef> _cache;
-  private final int capacity = 16 * 1024 * 1024;
 
   public Blur022StoredFieldsFormat(int chunkSize, CompressionMode compressionMode) {
     _chunkSize = chunkSize;
     _compressionMode = compressionMode;
-    _cache = new ConcurrentLinkedHashMap.Builder<CachedKey, BytesRef>().weigher(new
Weigher<BytesRef>() {
-      @Override
-      public int weightOf(BytesRef value) {
-        return value.bytes.length;
-      }
-    }).maximumWeightedCapacity(capacity).build();
   }
 
   static class CachedCompressionMode extends CompressionMode {
 
     final CompressionMode _compressionMode;
-    final SegmentInfo _si;
-    final ConcurrentLinkedHashMap<CachedKey, BytesRef> _cache;
 
-    CachedCompressionMode(CompressionMode compressionMode, SegmentInfo si,
-        ConcurrentLinkedHashMap<CachedKey, BytesRef> cache) {
+    CachedCompressionMode(CompressionMode compressionMode) {
       _compressionMode = compressionMode;
-      _si = si;
-      _cache = cache;
     }
 
     @Override
@@ -81,7 +64,7 @@ public final class Blur022StoredFieldsFormat extends StoredFieldsFormat
{
 
     @Override
     public Decompressor newDecompressor() {
-      return new CachedDecompressor(_compressionMode.newDecompressor(), _si, _cache);
+      return new CachedDecompressor(_compressionMode.newDecompressor());
     }
 
     @Override
@@ -94,7 +77,7 @@ public final class Blur022StoredFieldsFormat extends StoredFieldsFormat
{
   @Override
   public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos
fn, IOContext context)
       throws IOException {
-    CompressionMode compressionMode = new CachedCompressionMode(getCompressionMode(si), si,
_cache);
+    CompressionMode compressionMode = new CachedCompressionMode(getCompressionMode(si));
     return new CompressingStoredFieldsReader(directory, si, SEGMENT_SUFFIX, fn, context,
FORMAT_NAME, compressionMode);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9340af68/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java
b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java
index 66f2113..e837efc 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java
@@ -19,48 +19,22 @@ package org.apache.blur.lucene.codec;
 import java.io.IOException;
 
 import org.apache.lucene.codecs.compressing.Decompressor;
-import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 
-import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
-
 public class CachedDecompressor extends Decompressor {
 
   private final Decompressor _decompressor;
-
-  private final ThreadLocal<Entry> _entry = new ThreadLocal<Entry>();
-
-  static class Entry {
-    final IndexInput _indexInput;
-    final String _name;
-    final long _filePointer;
-    BytesRef _cache;
-
-    Entry(IndexInput indexInput, String name, long filePointer) {
-      _indexInput = indexInput;
-      _name = name;
-      _filePointer = filePointer;
+  private final ThreadLocal<Entry> _entry = new ThreadLocal<Entry>() {
+    @Override
+    protected Entry initialValue() {
+      return new Entry();
     }
+  };
 
-    boolean isValid(IndexInput indexInput, String name, long filePointer) {
-      if (_indexInput != indexInput) {
-        return false;
-      }
-      if (!_name.equals(name)) {
-        return false;
-      }
-      if (_filePointer != filePointer) {
-        return false;
-      }
-      return true;
-    }
-  }
-
-  public CachedDecompressor(Decompressor decompressor, SegmentInfo si,
-      ConcurrentLinkedHashMap<CachedKey, BytesRef> cache) {
+  public CachedDecompressor(Decompressor decompressor) {
     _decompressor = decompressor;
   }
 
@@ -73,9 +47,9 @@ public class CachedDecompressor extends Decompressor {
       long filePointer = indexInput.getFilePointer();
 
       Entry entry = _entry.get();
-      if (entry == null || !entry.isValid(indexInput, name, filePointer)) {
-        entry = new Entry(indexInput, name, filePointer);
-        entry._cache = new BytesRef(originalLength + 7);
+      if (!entry.isValid(indexInput, name, filePointer)) {
+        entry.setup(indexInput, name, filePointer);
+        entry._cache.grow(originalLength + 7);
         _decompressor.decompress(indexInput, originalLength, 0, originalLength, entry._cache);
         entry._cache.length = originalLength;
         entry._cache.offset = 0;
@@ -94,6 +68,32 @@ public class CachedDecompressor extends Decompressor {
 
   @Override
   public Decompressor clone() {
-    return this;
+    return new CachedDecompressor(_decompressor.clone());
+  }
+
+  static class Entry {
+    IndexInput _indexInput;
+    String _name;
+    long _filePointer = -1;
+    BytesRef _cache = new BytesRef();
+
+    void setup(IndexInput indexInput, String name, long filePointer) {
+      _indexInput = indexInput;
+      _name = name;
+      _filePointer = filePointer;
+    }
+
+    boolean isValid(IndexInput indexInput, String name, long filePointer) {
+      if (_indexInput != indexInput) {
+        return false;
+      }
+      if (!name.equals(_name)) {
+        return false;
+      }
+      if (_filePointer != filePointer) {
+        return false;
+      }
+      return true;
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9340af68/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java b/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
index 33d9161..5433432 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/warmup/IndexWarmup.java
@@ -16,6 +16,7 @@ package org.apache.blur.lucene.warmup;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -400,14 +401,17 @@ public class IndexWarmup {
     long s = System.nanoTime();
     double _targetThrouhput = _maxBytesPerSec / 1000.0 / 1000.0;
     for (Entry<String, OpenBitSet> e : filePartsToWarm.entrySet()) {
-      input = warmFile(directory, e.getKey(), e.getValue(), blockSize, context, bufferSize,
input);
-      if (input != null) {
-        long end = System.nanoTime();
-        double seconds = (end - s) / 1000000000.0;
-        double rateMbPerSec = (input.getTotalBytesRead() / seconds) / 1000 / 1000;
-        LOG.debug(
-            "Context [{0}] warming file [{1}] is [{2}%] complete " + "at rate of [{3} MB/s]
target was [{4} MB/s]",
-            context, e.getKey(), 100, rateMbPerSec, _targetThrouhput);
+      try {
+        input = warmFile(directory, e.getKey(), e.getValue(), blockSize, context, bufferSize,
input);
+        if (input != null) {
+          long end = System.nanoTime();
+          double seconds = (end - s) / 1000000000.0;
+          double rateMbPerSec = (input.getTotalBytesRead() / seconds) / 1000 / 1000;
+          LOG.debug("Context [{0}] warming file [{1}] is [{2}%] complete "
+              + "at rate of [{3} MB/s] target was [{4} MB/s]", context, e.getKey(), 100,
rateMbPerSec, _targetThrouhput);
+        }
+      } catch (FileNotFoundException ex) {
+        LOG.debug("File [{0}] no longer exists.", e.getKey());
       }
     }
     if (input != null) {


Mime
View raw message