incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: More updates the NRT code is still broken, but it's getting closer.
Date Tue, 19 Mar 2013 02:16:55 GMT
Updated Branches:
  refs/heads/0.1.5 a46014229 -> 8a0b462f6


More updates the NRT code is still broken, but it's getting closer.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/8a0b462f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/8a0b462f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/8a0b462f

Branch: refs/heads/0.1.5
Commit: 8a0b462f6fa2d2dd70770705f4d7b40f1e72e7a5
Parents: a460142
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Mar 18 22:15:34 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Mar 18 22:15:34 2013 -0400

----------------------------------------------------------------------
 .../org/apache/blur/analysis/BlurAnalyzer.java     |   57 +---
 .../org/apache/blur/analysis/DoubleAnalyzer.java   |   31 ++-
 .../org/apache/blur/analysis/FloatAnalyzer.java    |   42 ++-
 .../org/apache/blur/analysis/IntegerAnalyzer.java  |   42 ++-
 .../org/apache/blur/analysis/LongAnalyzer.java     |   42 ++-
 .../blur/lucene/search/AbstractWrapperQuery.java   |   18 +-
 .../org/apache/blur/lucene/search/FacetQuery.java  |   45 ++--
 .../apache/blur/lucene/search/FairSimilarity.java  |   17 +-
 .../apache/blur/lucene/search/IterablePaging.java  |   20 +-
 .../apache/blur/lucene/search/PagingCollector.java |    9 +-
 .../blur/lucene/search/StopExecutionCollector.java |    6 +-
 .../org/apache/blur/lucene/search/SuperParser.java |  114 +++---
 .../org/apache/blur/lucene/search/SuperQuery.java  |   51 ++--
 .../java/org/apache/blur/manager/IndexManager.java |  122 +++----
 .../java/org/apache/blur/manager/IndexServer.java  |    2 +-
 .../org/apache/blur/manager/QueryParserUtil.java   |    6 +-
 .../clusterstatus/ZookeeperClusterStatus.java      |    9 +-
 .../manager/indexserver/AbstractIndexServer.java   |  121 ++-----
 .../indexserver/DefaultBlurIndexWarmup.java        |  101 +++---
 .../indexserver/DistributedIndexServer.java        |  117 +++---
 .../blur/manager/indexserver/LocalIndexServer.java |   18 +-
 .../blur/manager/writer/AbstractBlurIndex.java     |   18 +-
 .../org/apache/blur/manager/writer/BlurIndex.java  |    2 +-
 .../blur/manager/writer/BlurIndexReader.java       |   15 +-
 .../apache/blur/manager/writer/BlurNRTIndex.java   |  115 +++---
 .../manager/writer/DirectoryReferenceCounter.java  |  279 ---------------
 .../manager/writer/DirectoryReferenceFileGC.java   |   98 -----
 .../blur/manager/writer/TransactionRecorder.java   |   35 ++-
 .../manager/writer/lucene/SnapshotIndexReader.java |  170 ---------
 .../writer/lucene/SoftDeleteIndexReader.java       |  213 -----------
 .../java/org/apache/blur/server/Configurable.java  |   25 ++
 .../java/org/apache/blur/server/Configured.java    |   42 +++
 .../java/org/apache/blur/server/ShardContext.java  |   87 +++++
 .../java/org/apache/blur/server/TableContext.java  |  165 +++++++++
 .../apache/blur/thrift/ThriftBlurShardServer.java  |   21 +-
 .../java/org/apache/blur/utils/BlurConstants.java  |    5 +-
 .../main/java/org/apache/blur/utils/BlurUtil.java  |  191 +++--------
 .../java/org/apache/blur/utils/PrimeDocCache.java  |   38 ++-
 .../utils/ResetableDocumentStoredFieldVisitor.java |  118 ++++++
 .../org/apache/blur/utils/RowDocumentUtil.java     |    4 +-
 .../java/org/apache/blur/utils/RowIndexWriter.java |    2 +-
 .../org/apache/blur/utils/TermDocIterable.java     |   48 ++-
 42 files changed, 1131 insertions(+), 1550 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java b/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
index e68d812..5f8f256 100644
--- a/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
@@ -16,7 +16,7 @@ package org.apache.blur.analysis;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import static org.apache.blur.lucene.LuceneConstant.LUCENE_VERSION;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 import static org.apache.blur.utils.BlurConstants.PRIME_DOC;
 import static org.apache.blur.utils.BlurConstants.RECORD_ID;
 import static org.apache.blur.utils.BlurConstants.ROW_ID;
@@ -28,7 +28,6 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.Reader;
 import java.lang.reflect.Constructor;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -45,12 +44,11 @@ import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.KeywordAnalyzer;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.AnalyzerWrapper;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Field.Index;
 import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.search.NumericRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Version;
@@ -59,7 +57,7 @@ import org.apache.thrift.protocol.TJSONProtocol;
 import org.apache.thrift.transport.TMemoryBuffer;
 import org.apache.thrift.transport.TMemoryInputTransport;
 
-public final class BlurAnalyzer extends Analyzer {
+public final class BlurAnalyzer extends AnalyzerWrapper {
 
   public enum TYPE {
     LONG, DOUBLE, FLOAT, INTEGER, DEFAULT
@@ -136,7 +134,7 @@ public final class BlurAnalyzer extends Analyzer {
     return type;
   }
 
-  public Query getNewRangeQuery(String field, String part1, String part2, boolean inclusive) {
+  public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
     Analyzer analyzer = getAnalyzer(field);
     if (analyzer instanceof LongAnalyzer) {
       LongAnalyzer a = (LongAnalyzer) analyzer;
@@ -144,26 +142,26 @@ public final class BlurAnalyzer extends Analyzer {
       int radix = a.getRadix();
       long min = Long.parseLong(part1, radix);
       long max = Long.parseLong(part2, radix);
-      return NumericRangeQuery.newLongRange(field, precisionStep, min, max, inclusive, inclusive);
+      return NumericRangeQuery.newLongRange(field, precisionStep, min, max, startInclusive, endInclusive);
     } else if (analyzer instanceof DoubleAnalyzer) {
       DoubleAnalyzer a = (DoubleAnalyzer) analyzer;
       int precisionStep = a.getPrecisionStep();
       double min = Double.parseDouble(part1);
       double max = Double.parseDouble(part2);
-      return NumericRangeQuery.newDoubleRange(field, precisionStep, min, max, inclusive, inclusive);
+      return NumericRangeQuery.newDoubleRange(field, precisionStep, min, max, startInclusive, endInclusive);
     } else if (analyzer instanceof FloatAnalyzer) {
       FloatAnalyzer a = (FloatAnalyzer) analyzer;
       int precisionStep = a.getPrecisionStep();
       float min = Float.parseFloat(part1);
       float max = Float.parseFloat(part2);
-      return NumericRangeQuery.newFloatRange(field, precisionStep, min, max, inclusive, inclusive);
+      return NumericRangeQuery.newFloatRange(field, precisionStep, min, max, startInclusive, endInclusive);
     } else if (analyzer instanceof IntegerAnalyzer) {
       IntegerAnalyzer a = (IntegerAnalyzer) analyzer;
       int precisionStep = a.getPrecisionStep();
       int radix = a.getRadix();
       int min = Integer.parseInt(part1, radix);
       int max = Integer.parseInt(part2, radix);
-      return NumericRangeQuery.newIntRange(field, precisionStep, min, max, inclusive, inclusive);
+      return NumericRangeQuery.newIntRange(field, precisionStep, min, max, startInclusive, endInclusive);
     }
     return null;
   }
@@ -228,42 +226,16 @@ public final class BlurAnalyzer extends Analyzer {
   public void close() {
 
   }
-
+  
   @Override
-  public TokenStream tokenStream(String fieldName, Reader reader) {
+  protected Analyzer getWrappedAnalyzer(String fieldName) {
     Analyzer analyzer = getAnalyzer(fieldName);
-    if (analyzer == null) {
-      analyzer = _defaultAnalyzer;
-    }
-
-    return analyzer.tokenStream(fieldName, reader);
+    return (analyzer != null) ? analyzer : _defaultAnalyzer;
   }
 
   @Override
-  public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
-    Analyzer analyzer = getAnalyzer(fieldName);
-    if (analyzer == null)
-      analyzer = _defaultAnalyzer;
-
-    return analyzer.reusableTokenStream(fieldName, reader);
-  }
-
-  /** Return the positionIncrementGap from the analyzer assigned to fieldName */
-  @Override
-  public int getPositionIncrementGap(String fieldName) {
-    Analyzer analyzer = getAnalyzer(fieldName);
-    if (analyzer == null)
-      analyzer = _defaultAnalyzer;
-    return analyzer.getPositionIncrementGap(fieldName);
-  }
-
-  /** Return the offsetGap from the analyzer assigned to field */
-  @Override
-  public int getOffsetGap(Fieldable field) {
-    Analyzer analyzer = getAnalyzer(field.name());
-    if (analyzer == null)
-      analyzer = _defaultAnalyzer;
-    return analyzer.getOffsetGap(field);
+  protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
+    return components;
   }
 
   public static BlurAnalyzer create(File file) throws IOException {
@@ -461,4 +433,5 @@ public final class BlurAnalyzer extends Analyzer {
     set.add(name);
   }
 
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/analysis/DoubleAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/DoubleAnalyzer.java b/src/blur-core/src/main/java/org/apache/blur/analysis/DoubleAnalyzer.java
index 92b4ccb..01abeae 100644
--- a/src/blur-core/src/main/java/org/apache/blur/analysis/DoubleAnalyzer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/analysis/DoubleAnalyzer.java
@@ -18,11 +18,11 @@ package org.apache.blur.analysis;
  */
 import java.io.IOException;
 import java.io.Reader;
-
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.NumericTokenStream;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.CharTokenizer;
 import org.apache.lucene.util.NumericUtils;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 
 public final class DoubleAnalyzer extends Analyzer {
 
@@ -44,18 +44,33 @@ public final class DoubleAnalyzer extends Analyzer {
   public void setPrecisionStep(int precisionStep) {
     this.precisionStep = precisionStep;
   }
-
   @Override
-  public TokenStream tokenStream(String fieldName, Reader reader) {
-    NumericTokenStream numericTokenStream = new NumericTokenStream(precisionStep);
+  protected TokenStreamComponents createComponents(String fieldName,
+      Reader reader) {
+    final CharTokenizer source = new CharTokenizer(LUCENE_VERSION, reader) {
+      @Override
+      protected boolean isTokenChar(int arg0) {
+        return true;
+      }
+    };
+
+    final double value;
     try {
-      numericTokenStream.setDoubleValue(toDouble(reader));
+      value = toDouble(reader);
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
-    return numericTokenStream;
-  }
+    final NumericTokenStream numericTokenStream = new NumericTokenStream(
+        precisionStep);
+    numericTokenStream.setDoubleValue(value);
 
+    return new TokenStreamComponents(source, numericTokenStream) {
+      public void setReader(Reader reader) throws IOException {
+        numericTokenStream.reset();
+        numericTokenStream.setDoubleValue(toDouble(reader));
+      }
+    };
+  }
   private double toDouble(Reader reader) throws IOException {
     StringBuilder builder = new StringBuilder(20);
     int read;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/analysis/FloatAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/FloatAnalyzer.java b/src/blur-core/src/main/java/org/apache/blur/analysis/FloatAnalyzer.java
index b2be140..fb04888 100644
--- a/src/blur-core/src/main/java/org/apache/blur/analysis/FloatAnalyzer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/analysis/FloatAnalyzer.java
@@ -21,8 +21,9 @@ import java.io.Reader;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.NumericTokenStream;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.CharTokenizer;
 import org.apache.lucene.util.NumericUtils;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 
 public final class FloatAnalyzer extends Analyzer {
 
@@ -45,17 +46,6 @@ public final class FloatAnalyzer extends Analyzer {
     this.precisionStep = precisionStep;
   }
 
-  @Override
-  public TokenStream tokenStream(String fieldName, Reader reader) {
-    NumericTokenStream numericTokenStream = new NumericTokenStream(precisionStep);
-    try {
-      numericTokenStream.setFloatValue(toFloat(reader));
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-    return numericTokenStream;
-  }
-
   private float toFloat(Reader reader) throws IOException {
     StringBuilder builder = new StringBuilder(20);
     int read;
@@ -64,5 +54,33 @@ public final class FloatAnalyzer extends Analyzer {
     }
     return Float.parseFloat(builder.toString());
   }
+  
+  @Override
+  protected TokenStreamComponents createComponents(String fieldName,
+      Reader reader) {
+    final CharTokenizer source = new CharTokenizer(LUCENE_VERSION, reader) {
+      @Override
+      protected boolean isTokenChar(int arg0) {
+        return true;
+      }
+    };
+
+    final float value;
+    try {
+      value = toFloat(reader);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    final NumericTokenStream numericTokenStream = new NumericTokenStream(
+        precisionStep);
+    numericTokenStream.setFloatValue(value);
+
+    return new TokenStreamComponents(source, numericTokenStream) {
+      public void setReader(Reader reader) throws IOException {
+        numericTokenStream.reset();
+        numericTokenStream.setFloatValue(toFloat(reader));
+      }
+    };
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/analysis/IntegerAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/IntegerAnalyzer.java b/src/blur-core/src/main/java/org/apache/blur/analysis/IntegerAnalyzer.java
index eb2c71d..f63330e 100644
--- a/src/blur-core/src/main/java/org/apache/blur/analysis/IntegerAnalyzer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/analysis/IntegerAnalyzer.java
@@ -21,8 +21,9 @@ import java.io.Reader;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.NumericTokenStream;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.CharTokenizer;
 import org.apache.lucene.util.NumericUtils;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 
 public final class IntegerAnalyzer extends Analyzer {
 
@@ -60,17 +61,6 @@ public final class IntegerAnalyzer extends Analyzer {
     this.precisionStep = precisionStep;
   }
 
-  @Override
-  public TokenStream tokenStream(String fieldName, Reader reader) {
-    NumericTokenStream numericTokenStream = new NumericTokenStream(precisionStep);
-    try {
-      numericTokenStream.setIntValue(toInteger(reader));
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-    return numericTokenStream;
-  }
-
   private int toInteger(Reader reader) throws IOException {
     StringBuilder builder = new StringBuilder(20);
     int read;
@@ -79,5 +69,33 @@ public final class IntegerAnalyzer extends Analyzer {
     }
     return Integer.parseInt(builder.toString(), radix);
   }
+  
+  @Override
+  protected TokenStreamComponents createComponents(String fieldName,
+      Reader reader) {
+    final CharTokenizer source = new CharTokenizer(LUCENE_VERSION, reader) {
+      @Override
+      protected boolean isTokenChar(int arg0) {
+        return true;
+      }
+    };
+
+    final int value;
+    try {
+      value = toInteger(reader);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    final NumericTokenStream numericTokenStream = new NumericTokenStream(
+        precisionStep);
+    numericTokenStream.setIntValue(value);
+
+    return new TokenStreamComponents(source, numericTokenStream) {
+      public void setReader(Reader reader) throws IOException {
+        numericTokenStream.reset();
+        numericTokenStream.setIntValue(toInteger(reader));
+      }
+    };
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/analysis/LongAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/LongAnalyzer.java b/src/blur-core/src/main/java/org/apache/blur/analysis/LongAnalyzer.java
index f5b8f3d..b8d7a54 100644
--- a/src/blur-core/src/main/java/org/apache/blur/analysis/LongAnalyzer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/analysis/LongAnalyzer.java
@@ -21,8 +21,9 @@ import java.io.Reader;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.NumericTokenStream;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.CharTokenizer;
 import org.apache.lucene.util.NumericUtils;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 
 public final class LongAnalyzer extends Analyzer {
 
@@ -60,17 +61,6 @@ public final class LongAnalyzer extends Analyzer {
     this.precisionStep = precisionStep;
   }
 
-  @Override
-  public TokenStream tokenStream(String fieldName, Reader reader) {
-    NumericTokenStream numericTokenStream = new NumericTokenStream(precisionStep);
-    try {
-      numericTokenStream.setLongValue(toLong(reader));
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-    return numericTokenStream;
-  }
-
   private long toLong(Reader reader) throws IOException {
     StringBuilder builder = new StringBuilder(20);
     int read;
@@ -79,5 +69,33 @@ public final class LongAnalyzer extends Analyzer {
     }
     return Long.parseLong(builder.toString(), radix);
   }
+  
+  @Override
+  protected TokenStreamComponents createComponents(String fieldName,
+      Reader reader) {
+    final CharTokenizer source = new CharTokenizer(LUCENE_VERSION, reader) {
+      @Override
+      protected boolean isTokenChar(int arg0) {
+        return true;
+      }
+    };
+
+    final long value;
+    try {
+      value = toLong(reader);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    final NumericTokenStream numericTokenStream = new NumericTokenStream(
+        precisionStep);
+    numericTokenStream.setLongValue(value);
+
+    return new TokenStreamComponents(source, numericTokenStream) {
+      public void setReader(Reader reader) throws IOException {
+        numericTokenStream.reset();
+        numericTokenStream.setLongValue(toLong(reader));
+      }
+    };
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java b/src/blur-core/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java
index 5fa4116..f9c4c6b 100644
--- a/src/blur-core/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java
+++ b/src/blur-core/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java
@@ -21,14 +21,12 @@ import java.util.Set;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Searcher;
-import org.apache.lucene.search.Similarity;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.similarities.Similarity;
 
-@SuppressWarnings("deprecation")
 public abstract class AbstractWrapperQuery extends Query {
-  private static final long serialVersionUID = -4512813621542220044L;
   protected Query _query;
   protected boolean _rewritten;
 
@@ -41,13 +39,9 @@ public abstract class AbstractWrapperQuery extends Query {
     this._rewritten = rewritten;
   }
 
-  public abstract Object clone();
+  public abstract Query clone();
 
-  public Query combine(Query[] queries) {
-    return _query.combine(queries);
-  }
-
-  public abstract Weight createWeight(Searcher searcher) throws IOException;
+  public abstract Weight createWeight(IndexSearcher searcher) throws IOException;
 
   public boolean equals(Object obj) {
     return _query.equals(obj);
@@ -61,8 +55,8 @@ public abstract class AbstractWrapperQuery extends Query {
     return _query.getBoost();
   }
 
-  public Similarity getSimilarity(Searcher searcher) {
-    return _query.getSimilarity(searcher);
+  public Similarity getSimilarity(IndexSearcher searcher) {
+    return searcher.getSimilarity();
   }
 
   public int hashCode() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/lucene/search/FacetQuery.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/lucene/search/FacetQuery.java b/src/blur-core/src/main/java/org/apache/blur/lucene/search/FacetQuery.java
index 980493c..1841fe9 100644
--- a/src/blur-core/src/main/java/org/apache/blur/lucene/search/FacetQuery.java
+++ b/src/blur-core/src/main/java/org/apache/blur/lucene/search/FacetQuery.java
@@ -19,17 +19,17 @@ package org.apache.blur.lucene.search;
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicLongArray;
 
+import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Searcher;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.Bits;
 
-@SuppressWarnings("deprecation")
 public class FacetQuery extends AbstractWrapperQuery {
 
-  private static final long serialVersionUID = -9131606859383383004L;
   private Query[] facets;
   private AtomicLongArray counts;
 
@@ -54,7 +54,7 @@ public class FacetQuery extends AbstractWrapperQuery {
   }
 
   @Override
-  public Object clone() {
+  public Query clone() {
     return new FacetQuery((Query) _query.clone(), facets, counts, _rewritten);
   }
 
@@ -70,12 +70,12 @@ public class FacetQuery extends AbstractWrapperQuery {
   }
 
   @Override
-  public Weight createWeight(Searcher searcher) throws IOException {
+  public Weight createWeight(IndexSearcher searcher) throws IOException {
     Weight weight = _query.createWeight(searcher);
     return new FacetWeight(weight, getWeights(searcher), counts);
   }
 
-  private Weight[] getWeights(Searcher searcher) throws IOException {
+  private Weight[] getWeights(IndexSearcher searcher) throws IOException {
     Weight[] weights = new Weight[facets.length];
     for (int i = 0; i < weights.length; i++) {
       weights[i] = facets[i].createWeight(searcher);
@@ -85,7 +85,6 @@ public class FacetQuery extends AbstractWrapperQuery {
 
   public static class FacetWeight extends Weight {
 
-    private static final long serialVersionUID = -5649908738708119094L;
     private Weight weight;
     private Weight[] facets;
     private AtomicLongArray counts;
@@ -97,7 +96,7 @@ public class FacetQuery extends AbstractWrapperQuery {
     }
 
     @Override
-    public Explanation explain(IndexReader reader, int doc) throws IOException {
+    public Explanation explain(AtomicReaderContext reader, int doc) throws IOException {
       return weight.explain(reader, doc);
     }
 
@@ -107,35 +106,30 @@ public class FacetQuery extends AbstractWrapperQuery {
     }
 
     @Override
-    public float getValue() {
-      return weight.getValue();
+    public void normalize(float norm, float topLevelBoost) {
+      weight.normalize(norm, topLevelBoost);
     }
 
     @Override
-    public void normalize(float norm) {
-      weight.normalize(norm);
-    }
-
-    @Override
-    public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
-      Scorer scorer = weight.scorer(reader, true, topScorer);
+    public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, Bits acceptDocs) throws IOException {
+      Scorer scorer = weight.scorer(context, true, topScorer, acceptDocs);
       if (scorer == null) {
         return null;
       }
-      return new FacetScorer(scorer, getScorers(reader, true, topScorer), counts);
+      return new FacetScorer(scorer, getScorers(context, true, topScorer, acceptDocs), counts);
     }
 
-    private Scorer[] getScorers(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
+    private Scorer[] getScorers(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, Bits acceptDocs) throws IOException {
       Scorer[] scorers = new Scorer[facets.length];
       for (int i = 0; i < scorers.length; i++) {
-        scorers[i] = facets[i].scorer(reader, scoreDocsInOrder, topScorer);
+        scorers[i] = facets[i].scorer(context, scoreDocsInOrder, topScorer, acceptDocs);
       }
       return scorers;
     }
 
     @Override
-    public float sumOfSquaredWeights() throws IOException {
-      return weight.sumOfSquaredWeights();
+    public float getValueForNormalization() throws IOException {
+      return weight.getValueForNormalization();
     }
   }
 
@@ -147,7 +141,7 @@ public class FacetQuery extends AbstractWrapperQuery {
     private int facetLength;
 
     public FacetScorer(Scorer scorer, Scorer[] facets, AtomicLongArray counts) {
-      super(scorer.getSimilarity());
+      super(scorer.getWeight());
       this.baseScorer = scorer;
       this.facets = facets;
       this.counts = counts;
@@ -197,5 +191,10 @@ public class FacetQuery extends AbstractWrapperQuery {
     public int nextDoc() throws IOException {
       return processFacets(baseScorer.nextDoc());
     }
+
+    @Override
+    public int freq() throws IOException {
+      return baseScorer.freq();
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java b/src/blur-core/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
index f0203d1..8c8b945 100644
--- a/src/blur-core/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
+++ b/src/blur-core/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
@@ -17,11 +17,10 @@ package org.apache.blur.lucene.search;
  * limitations under the License.
  */
 import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.TFIDFSimilarity;
+import org.apache.lucene.util.BytesRef;
 
-public class FairSimilarity extends Similarity {
-
-  private static final long serialVersionUID = 8819964136561756067L;
+public class FairSimilarity extends TFIDFSimilarity {
 
   @Override
   public float coord(int overlap, int maxOverlap) {
@@ -29,7 +28,7 @@ public class FairSimilarity extends Similarity {
   }
 
   @Override
-  public float idf(int docFreq, int numDocs) {
+  public float idf(long docFreq, long numDocs) {
     return 1;
   }
 
@@ -49,8 +48,14 @@ public class FairSimilarity extends Similarity {
   }
 
   @Override
-  public float computeNorm(String field, FieldInvertState state) {
+  public float scorePayload(int doc, int start, int end, BytesRef payload) {
     return 1;
   }
 
+  @Override
+  public float lengthNorm(FieldInvertState fieldInvertState) {
+    throw new RuntimeException("not sure");
+//    return 0;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/lucene/search/IterablePaging.java b/src/blur-core/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
index 06a106b..4e2cbc5 100644
--- a/src/blur-core/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
+++ b/src/blur-core/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
@@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopScoreDocCollector;
 
 /**
  * The {@link IterablePaging} class allows for easy paging through lucene hits.
@@ -159,11 +160,11 @@ public class IterablePaging implements Iterable<ScoreDoc> {
   }
 
   class PagingIterator implements Iterator<ScoreDoc> {
-    private PagingCollector collector;
     private ScoreDoc[] scoreDocs;
     private int counter = 0;
     private int offset = 0;
     private int endPosition = gather == -1 ? Integer.MAX_VALUE : skipTo + gather;
+    private ScoreDoc lastScoreDoc;
 
     PagingIterator() {
       search();
@@ -172,20 +173,21 @@ public class IterablePaging implements Iterable<ScoreDoc> {
     void search() {
       long s = System.currentTimeMillis();
       progressRef.searchesPerformed.incrementAndGet();
-      if (collector == null) {
-        collector = new PagingCollector(numHitsToCollect);
-      } else {
-        collector = new PagingCollector(numHitsToCollect, scoreDocs[scoreDocs.length - 1]);
-      }
       try {
+        TopScoreDocCollector collector = TopScoreDocCollector.create(numHitsToCollect, lastScoreDoc, true);
         StopExecutionCollector stopExecutionCollector = new StopExecutionCollector(collector, running);
         searcher.search(query, stopExecutionCollector);
+        totalHitsRef.totalHits.set(collector.getTotalHits());
+        scoreDocs = collector.topDocs().scoreDocs;
       } catch (IOException e) {
+        e.printStackTrace();
         throw new RuntimeException(e);
       }
-
-      totalHitsRef.totalHits.set(collector.getTotalHits());
-      scoreDocs = collector.topDocs().scoreDocs;
+      if (scoreDocs.length > 0) {
+        lastScoreDoc = scoreDocs[scoreDocs.length - 1];
+      } else {
+        lastScoreDoc = null;
+      }
       long e = System.currentTimeMillis();
       progressRef.queryTime.addAndGet(e - s);
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/lucene/search/PagingCollector.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/lucene/search/PagingCollector.java b/src/blur-core/src/main/java/org/apache/blur/lucene/search/PagingCollector.java
index 2bead97..385c29d 100644
--- a/src/blur-core/src/main/java/org/apache/blur/lucene/search/PagingCollector.java
+++ b/src/blur-core/src/main/java/org/apache/blur/lucene/search/PagingCollector.java
@@ -18,7 +18,7 @@ package org.apache.blur.lucene.search;
  */
 import java.io.IOException;
 
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TopDocs;
@@ -76,8 +76,8 @@ public class PagingCollector extends TopDocsCollector<ScoreDoc> {
   }
 
   @Override
-  public void setNextReader(IndexReader reader, int docBase) throws IOException {
-    this.docBase = docBase;
+  public void setNextReader(AtomicReaderContext context) throws IOException {
+    this.docBase = context.docBase;
   }
 
   @Override
@@ -98,8 +98,8 @@ public class PagingCollector extends TopDocsCollector<ScoreDoc> {
     private boolean prePopulate;
 
     HitQueue(int size, boolean prePopulate) {
+      super(size);
       this.prePopulate = prePopulate;
-      initialize(size);
     }
 
     @Override
@@ -116,4 +116,5 @@ public class PagingCollector extends TopDocsCollector<ScoreDoc> {
       }
     }
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/lucene/search/StopExecutionCollector.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/lucene/search/StopExecutionCollector.java b/src/blur-core/src/main/java/org/apache/blur/lucene/search/StopExecutionCollector.java
index 7a8eebe..d97a14f 100644
--- a/src/blur-core/src/main/java/org/apache/blur/lucene/search/StopExecutionCollector.java
+++ b/src/blur-core/src/main/java/org/apache/blur/lucene/search/StopExecutionCollector.java
@@ -20,7 +20,7 @@ import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.Scorer;
 
@@ -56,8 +56,8 @@ public class StopExecutionCollector extends Collector {
     _collector.collect(doc);
   }
 
-  public void setNextReader(IndexReader reader, int docBase) throws IOException {
-    _collector.setNextReader(reader, docBase);
+  public void setNextReader(AtomicReaderContext context) throws IOException {
+    _collector.setNextReader(context);
   }
 
   public void setScorer(Scorer scorer) throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperParser.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperParser.java b/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperParser.java
index 296cc0b..b97124f 100644
--- a/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperParser.java
+++ b/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperParser.java
@@ -27,8 +27,8 @@ import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.analysis.BlurAnalyzer.TYPE;
 import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Filter;
@@ -80,7 +80,6 @@ public class SuperParser extends QueryParser {
   @Override
   protected MultiPhraseQuery newMultiPhraseQuery() {
     return new MultiPhraseQuery() {
-      private static final long serialVersionUID = 2743009696906520410L;
 
       @Override
       public void add(Term[] terms, int position) {
@@ -95,7 +94,6 @@ public class SuperParser extends QueryParser {
   @Override
   protected PhraseQuery newPhraseQuery() {
     return new PhraseQuery() {
-      private static final long serialVersionUID = 1927750709523859808L;
 
       @Override
       public void add(Term term, int position) {
@@ -116,18 +114,18 @@ public class SuperParser extends QueryParser {
   }
 
   @Override
-  protected Query newRangeQuery(String field, String part1, String part2, boolean inclusive) {
-    Query q = blurAnalyzer.getNewRangeQuery(field, part1, part2, inclusive);
+  protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
+    Query q = blurAnalyzer.getNewRangeQuery(field, part1, part2, startInclusive, endInclusive);
     if (q != null) {
       return addField(q, field);
     }
-    return addField(super.newRangeQuery(field, part1, part2, inclusive), field);
+    return addField(super.newRangeQuery(field, part1, part2, startInclusive, endInclusive), field);
   }
 
   @Override
   protected Query newTermQuery(Term term) {
     String field = term.field();
-    Query q = blurAnalyzer.getNewRangeQuery(field, term.text(), term.text(), true);
+    Query q = blurAnalyzer.getNewRangeQuery(field, term.text(), term.text(), true, true);
     if (q != null) {
       return addField(q, field);
     }
@@ -185,56 +183,56 @@ public class SuperParser extends QueryParser {
     return new FilteredQuery(query, queryFilter);
   }
 
-//  private boolean isSameGroupName(BooleanQuery booleanQuery) {
-//    String groupName = findFirstGroupName(booleanQuery);
-//    if (groupName == null) {
-//      return false;
-//    }
-//    return isSameGroupName(booleanQuery, groupName);
-//  }
-//
-//  private boolean isSameGroupName(Query query, String groupName) {
-//    if (query instanceof BooleanQuery) {
-//      BooleanQuery booleanQuery = (BooleanQuery) query;
-//      for (BooleanClause clause : booleanQuery.clauses()) {
-//        if (!isSameGroupName(clause.getQuery(), groupName)) {
-//          return false;
-//        }
-//      }
-//      return true;
-//    } else {
-//      String fieldName = fieldNames.get(query);
-//      String currentGroupName = getGroupName(fieldName);
-//      if (groupName.equals(currentGroupName)) {
-//        return true;
-//      }
-//      return false;
-//    }
-//  }
-//
-//  private String getGroupName(String fieldName) {
-//    if (fieldName == null) {
-//      return null;
-//    }
-//    int index = fieldName.indexOf(SEP);
-//    if (index < 0) {
-//      return null;
-//    }
-//    return fieldName.substring(0, index);
-//  }
-//
-//  private String findFirstGroupName(Query query) {
-//    if (query instanceof BooleanQuery) {
-//      BooleanQuery booleanQuery = (BooleanQuery) query;
-//      for (BooleanClause clause : booleanQuery.clauses()) {
-//        return findFirstGroupName(clause.getQuery());
-//      }
-//      return null;
-//    } else {
-//      String fieldName = fieldNames.get(query);
-//      return getGroupName(fieldName);
-//    }
-//  }
+  // private boolean isSameGroupName(BooleanQuery booleanQuery) {
+  // String groupName = findFirstGroupName(booleanQuery);
+  // if (groupName == null) {
+  // return false;
+  // }
+  // return isSameGroupName(booleanQuery, groupName);
+  // }
+  //
+  // private boolean isSameGroupName(Query query, String groupName) {
+  // if (query instanceof BooleanQuery) {
+  // BooleanQuery booleanQuery = (BooleanQuery) query;
+  // for (BooleanClause clause : booleanQuery.clauses()) {
+  // if (!isSameGroupName(clause.getQuery(), groupName)) {
+  // return false;
+  // }
+  // }
+  // return true;
+  // } else {
+  // String fieldName = fieldNames.get(query);
+  // String currentGroupName = getGroupName(fieldName);
+  // if (groupName.equals(currentGroupName)) {
+  // return true;
+  // }
+  // return false;
+  // }
+  // }
+  //
+  // private String getGroupName(String fieldName) {
+  // if (fieldName == null) {
+  // return null;
+  // }
+  // int index = fieldName.indexOf(SEP);
+  // if (index < 0) {
+  // return null;
+  // }
+  // return fieldName.substring(0, index);
+  // }
+  //
+  // private String findFirstGroupName(Query query) {
+  // if (query instanceof BooleanQuery) {
+  // BooleanQuery booleanQuery = (BooleanQuery) query;
+  // for (BooleanClause clause : booleanQuery.clauses()) {
+  // return findFirstGroupName(clause.getQuery());
+  // }
+  // return null;
+  // } else {
+  // String fieldName = fieldNames.get(query);
+  // return getGroupName(fieldName);
+  // }
+  // }
 
   private Query addField(Query q, String field) {
     fieldNames.put(q, field);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperQuery.java b/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
index bd83748..fc24d21 100644
--- a/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
+++ b/src/blur-core/src/main/java/org/apache/blur/lucene/search/SuperQuery.java
@@ -20,21 +20,20 @@ import java.io.IOException;
 
 import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.blur.utils.PrimeDocCache;
+import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Searcher;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.OpenBitSet;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
-@SuppressWarnings("deprecation")
 public class SuperQuery extends AbstractWrapperQuery {
 
-  private static final long serialVersionUID = -5901574044714034398L;
   private ScoreType scoreType;
 
   public SuperQuery(Query query, ScoreType scoreType) {
@@ -47,11 +46,11 @@ public class SuperQuery extends AbstractWrapperQuery {
     this.scoreType = scoreType;
   }
 
-  public Object clone() {
+  public Query clone() {
     return new SuperQuery((Query) _query.clone(), scoreType, _rewritten);
   }
 
-  public Weight createWeight(Searcher searcher) throws IOException {
+  public Weight createWeight(IndexSearcher searcher) throws IOException {
     Weight weight = _query.createWeight(searcher);
     return new SuperWeight(weight, _query.toString(), this, scoreType);
   }
@@ -73,8 +72,6 @@ public class SuperQuery extends AbstractWrapperQuery {
 
   public static class SuperWeight extends Weight {
 
-    private static final long serialVersionUID = -4832849792097064960L;
-
     private Weight weight;
     private String originalQueryStr;
     private Query query;
@@ -88,39 +85,42 @@ public class SuperQuery extends AbstractWrapperQuery {
     }
 
     @Override
-    public Explanation explain(IndexReader reader, int doc) throws IOException {
-      throw new RuntimeException("not supported");
-    }
-
-    @Override
     public Query getQuery() {
       return query;
     }
 
     @Override
-    public float getValue() {
-      return weight.getValue();
+    public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
+      throw new RuntimeException("not supported");
     }
 
+    /*
+     * This method needs to implement in some other way Weight doesn't provide
+     * this method at all
+     * 
+     * @Override public float getValue() { return weight.getValue(); }
+     */
+
     @Override
-    public void normalize(float norm) {
-      weight.normalize(norm);
+    public void normalize(float norm, float topLevelBoost) {
+      weight.normalize(norm, topLevelBoost);
     }
 
     @Override
-    public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
-      Scorer scorer = weight.scorer(reader, true, topScorer);
+    public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, Bits acceptDocs) throws IOException {
+      Scorer scorer = weight.scorer(context, true, topScorer, acceptDocs);
       if (scorer == null) {
         return null;
       }
-      OpenBitSet primeDocBitSet = PrimeDocCache.getPrimeDocBitSet(reader);
+      OpenBitSet primeDocBitSet = PrimeDocCache.getPrimeDocBitSet(context.reader());
       return new SuperScorer(scorer, primeDocBitSet, originalQueryStr, scoreType);
     }
 
     @Override
-    public float sumOfSquaredWeights() throws IOException {
-      return weight.sumOfSquaredWeights();
+    public float getValueForNormalization() throws IOException {
+      return weight.getValueForNormalization();
     }
+
   }
 
   @SuppressWarnings("unused")
@@ -146,7 +146,7 @@ public class SuperQuery extends AbstractWrapperQuery {
     private int hitsInEntity;
 
     protected SuperScorer(Scorer scorer, OpenBitSet bitSet, String originalQueryStr, ScoreType scoreType) {
-      super(scorer.getSimilarity());
+      super(scorer.getWeight());
       this.scorer = scorer;
       this.bitSet = bitSet;
       this.originalQueryStr = originalQueryStr;
@@ -263,6 +263,11 @@ public class SuperQuery extends AbstractWrapperQuery {
     private boolean isScorerExhausted(int doc) {
       return doc == NO_MORE_DOCS ? true : false;
     }
+
+    @Override
+    public int freq() throws IOException {
+      return scorer.freq();
+    }
   }
 
   public Query getQuery() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 725174a..8a6fca2 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -20,8 +20,6 @@ import static org.apache.blur.utils.BlurConstants.PRIME_DOC;
 import static org.apache.blur.utils.BlurConstants.RECORD_ID;
 import static org.apache.blur.utils.BlurConstants.ROW_ID;
 import static org.apache.blur.utils.BlurUtil.findRecordMutation;
-import static org.apache.blur.utils.BlurUtil.readFilter;
-import static org.apache.blur.utils.BlurUtil.readQuery;
 import static org.apache.blur.utils.RowDocumentUtil.getColumns;
 import static org.apache.blur.utils.RowDocumentUtil.getRow;
 
@@ -48,7 +46,6 @@ import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
-import org.apache.blur.lucene.EscapeRewrite;
 import org.apache.blur.lucene.search.FacetQuery;
 import org.apache.blur.manager.results.BlurResultIterable;
 import org.apache.blur.manager.results.BlurResultIterableSearcher;
@@ -80,22 +77,24 @@ import org.apache.blur.thrift.generated.SimpleQuery;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurExecutorCompletionService;
 import org.apache.blur.utils.BlurExecutorCompletionService.Cancel;
+import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.utils.ForkJoin;
 import org.apache.blur.utils.ForkJoin.Merger;
 import org.apache.blur.utils.ForkJoin.ParallelCall;
-import org.apache.blur.utils.TermDocIterable;
+import org.apache.blur.utils.ResetableDocumentStoredFieldVisitor;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.FieldSelectorResult;
+import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Filter;
@@ -104,7 +103,8 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 
 public class IndexManager {
 
@@ -348,11 +348,11 @@ public class IndexManager {
   }
 
   private Filter getFilter(ExpertQuery expertQuery) throws BException {
-    return readFilter(expertQuery.getFilter());
+    throw new BException("Expert query not supported", new Throwable());
   }
 
   private Query getQuery(ExpertQuery expertQuery) throws BException {
-    return readQuery(expertQuery.getQuery());
+    throw new BException("Expert query not supported", new Throwable());
   }
 
   private boolean isSimpleQuery(BlurQuery blurQuery) {
@@ -409,21 +409,23 @@ public class IndexManager {
     if (docId >= reader.maxDoc()) {
       throw new RuntimeException("Location id [" + locationId + "] with docId [" + docId + "] is not valid.");
     }
+    Bits liveDocs = MultiFields.getLiveDocs(reader);
     if (selector.isRecordOnly()) {
       // select only the row for the given data or location id.
-      if (reader.isDeleted(docId)) {
+      if (liveDocs != null && !liveDocs.get(docId)) {
         fetchResult.exists = false;
         fetchResult.deleted = true;
         return;
       } else {
         fetchResult.exists = true;
         fetchResult.deleted = false;
-        Document document = reader.document(docId, getFieldSelector(selector));
+        reader.document(docId, getFieldSelector(selector));
+        Document document = reader.document(docId);
         fetchResult.recordResult = getColumns(document);
         return;
       }
     } else {
-      if (reader.isDeleted(docId)) {
+      if (liveDocs != null && !liveDocs.get(docId)) {
         fetchResult.exists = false;
         fetchResult.deleted = true;
         return;
@@ -431,26 +433,25 @@ public class IndexManager {
         fetchResult.exists = true;
         fetchResult.deleted = false;
         String rowId = getRowId(reader, docId);
-        TermDocs termDocs = reader.termDocs(new Term(ROW_ID, rowId));
-        fetchResult.rowResult = new FetchRowResult(getRow(new TermDocIterable(termDocs, reader, getFieldSelector(selector))));
+
+        List<Document> docs = BlurUtil.termSearch(reader, new Term(ROW_ID, rowId), getFieldSelector(selector));
+        fetchResult.rowResult = new FetchRowResult(getRow(docs));
         return;
       }
     }
   }
 
   private static String getRowId(IndexReader reader, int docId) throws CorruptIndexException, IOException {
-    Document document = reader.document(docId, new FieldSelector() {
-      private static final long serialVersionUID = 4912420100148752051L;
-
+    reader.document(docId, new StoredFieldVisitor() {
       @Override
-      public FieldSelectorResult accept(String fieldName) {
-        if (ROW_ID.equals(fieldName)) {
-          return FieldSelectorResult.LOAD_AND_BREAK;
+      public Status needsField(FieldInfo fieldInfo) throws IOException {
+        if (ROW_ID.equals(fieldInfo.name)) {
+          return StoredFieldVisitor.Status.STOP;
         }
-        return FieldSelectorResult.NO_LOAD;
+        return StoredFieldVisitor.Status.NO;
       }
     });
-    return document.get(ROW_ID);
+    return reader.document(docId).get(ROW_ID);
   }
 
   private static String getColumnName(String fieldName) {
@@ -461,39 +462,38 @@ public class IndexManager {
     return fieldName.substring(0, fieldName.lastIndexOf('.'));
   }
 
-  public static FieldSelector getFieldSelector(final Selector selector) {
-    return new FieldSelector() {
-      private static final long serialVersionUID = 4089164344758433000L;
-
+  public static ResetableDocumentStoredFieldVisitor getFieldSelector(final Selector selector) {
+    return new ResetableDocumentStoredFieldVisitor() {
       @Override
-      public FieldSelectorResult accept(String fieldName) {
-        if (ROW_ID.equals(fieldName)) {
-          return FieldSelectorResult.LOAD;
+      public Status needsField(FieldInfo fieldInfo) throws IOException {
+        if (ROW_ID.equals(fieldInfo.name)) {
+          return StoredFieldVisitor.Status.YES;
         }
-        if (RECORD_ID.equals(fieldName)) {
-          return FieldSelectorResult.LOAD;
+        if (RECORD_ID.equals(fieldInfo.name)) {
+          return StoredFieldVisitor.Status.YES;
         }
-        if (PRIME_DOC.equals(fieldName)) {
-          return FieldSelectorResult.NO_LOAD;
+        if (PRIME_DOC.equals(fieldInfo.name)) {
+          return StoredFieldVisitor.Status.NO;
         }
         if (selector.columnFamiliesToFetch == null && selector.columnsToFetch == null) {
-          return FieldSelectorResult.LOAD;
+          return StoredFieldVisitor.Status.YES;
         }
-        String columnFamily = getColumnFamily(fieldName);
+        String columnFamily = getColumnFamily(fieldInfo.name);
         if (selector.columnFamiliesToFetch != null) {
           if (selector.columnFamiliesToFetch.contains(columnFamily)) {
-            return FieldSelectorResult.LOAD;
+            return StoredFieldVisitor.Status.YES;
           }
         }
-        String columnName = getColumnName(fieldName);
+        String columnName = getColumnName(fieldInfo.name);
         if (selector.columnsToFetch != null) {
           Set<String> columns = selector.columnsToFetch.get(columnFamily);
           if (columns != null && columns.contains(columnName)) {
-            return FieldSelectorResult.LOAD;
+            return StoredFieldVisitor.Status.YES;
           }
         }
-        return FieldSelectorResult.NO_LOAD;
+        return StoredFieldVisitor.Status.NO;
       }
+
     };
   }
 
@@ -577,27 +577,18 @@ public class IndexManager {
 
   public static List<String> terms(IndexReader reader, String columnFamily, String columnName, String startWith, short size) throws IOException {
     Term term = getTerm(columnFamily, columnName, startWith);
-    String field = term.field();
     List<String> terms = new ArrayList<String>(size);
-    TermEnum termEnum = reader.terms(term);
-    try {
-      do {
-        Term currentTerm = termEnum.term();
-        if (currentTerm == null) {
-          return terms;
-        }
-        if (!currentTerm.field().equals(field)) {
-          break;
-        }
-        terms.add(currentTerm.text());
-        if (terms.size() >= size) {
-          return terms;
-        }
-      } while (termEnum.next());
-      return terms;
-    } finally {
-      termEnum.close();
+    AtomicReader areader = BlurUtil.getAtomicReader(reader);
+    Terms termsAll = areader.terms(term.field());
+    TermsEnum termEnum = termsAll.iterator(null);
+    BytesRef currentTermText;
+    while ((currentTermText = termEnum.next()) != null) {
+      terms.add(currentTermText.utf8ToString());
+      if (terms.size() >= size) {
+        return terms;
+      }
     }
+    return terms;
   }
 
   private static Term getTerm(String columnFamily, String columnName, String value) {
@@ -617,7 +608,7 @@ public class IndexManager {
     for (BlurIndex blurIndex : blurIndexes.values()) {
       IndexReader reader = blurIndex.getIndexReader();
       try {
-        FieldInfos mergedFieldInfos = ReaderUtil.getMergedFieldInfos(reader);
+        FieldInfos mergedFieldInfos = MultiFields.getMergedFieldInfos(reader);
         for (FieldInfo fieldInfo : mergedFieldInfos) {
           String fieldName = fieldInfo.name;
           int index = fieldName.indexOf('.');
@@ -949,8 +940,10 @@ public class IndexManager {
         BlurIndex index = entry.getValue();
         IndexReader reader = index.getIndexReader();
         String shard = entry.getKey();
-        IndexReader escapeReader = EscapeRewrite.wrap(reader, _running);
-        IndexSearcher searcher = new IndexSearcher(escapeReader);
+        // @TODO need to add escapable rewriter
+        // IndexReader escapeReader = EscapeRewrite.wrap(reader, _running);
+        // IndexSearcher searcher = new IndexSearcher(escapeReader);
+        IndexSearcher searcher = new IndexSearcher(reader);
         searcher.setSimilarity(_indexServer.getSimilarity(_table));
         Query rewrite = searcher.rewrite((Query) _query.clone());
         return new BlurResultIterableSearcher(_running, rewrite, _table, shard, searcher, _selector, reader);
@@ -992,5 +985,4 @@ public class IndexManager {
       }
     }
   }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
index 2a3f937..1097048 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
@@ -24,7 +24,7 @@ import java.util.SortedSet;
 import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.Similarity;
 
 
 public interface IndexServer {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java b/src/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
index 3c4d92c..469431a 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
@@ -16,13 +16,13 @@ package org.apache.blur.manager;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import static org.apache.blur.lucene.LuceneConstant.LUCENE_VERSION;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 
 import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.lucene.search.SuperParser;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.ScoreType;
-import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.FilteredQuery;
 import org.apache.lucene.search.Query;
@@ -30,7 +30,7 @@ import org.apache.lucene.search.QueryWrapperFilter;
 
 public class QueryParserUtil {
 
-  public static Query parseQuery(String query, boolean superQueryOn, BlurAnalyzer analyzer, Filter postFilter, Filter preFilter, ScoreType scoreType) throws ParseException {
+  public static Query parseQuery(String query, boolean superQueryOn, BlurAnalyzer analyzer, Filter postFilter, Filter preFilter, ScoreType scoreType) throws ParseException   {
     Query result = new SuperParser(LUCENE_VERSION, analyzer, superQueryOn, preFilter, scoreType).parse(query);
     if (postFilter == null) {
       return result;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java b/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
index 81ea5f4..0c595f4 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
@@ -40,13 +40,13 @@ import org.apache.blur.thrift.generated.ColumnPreCache;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.zookeeper.WatchChildren;
+import org.apache.blur.zookeeper.WatchChildren.OnChange;
 import org.apache.blur.zookeeper.WatchNodeData;
 import org.apache.blur.zookeeper.WatchNodeExistance;
 import org.apache.blur.zookeeper.ZkUtils;
-import org.apache.blur.zookeeper.WatchChildren.OnChange;
 import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.DeflateCodec;
-import org.apache.lucene.search.Similarity;
+import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.lucene.search.similarities.Similarity;
 import org.apache.thrift.TBase;
 import org.apache.thrift.TException;
 import org.apache.thrift.protocol.TJSONProtocol;
@@ -59,7 +59,6 @@ import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.Stat;
 
-
 public class ZookeeperClusterStatus extends ClusterStatus {
 
   private static final Log LOG = LogFactory.getLog(ZookeeperClusterStatus.class);
@@ -671,7 +670,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
     try {
       checkIfOpen();
       if (tableDescriptor.getCompressionClass() == null) {
-        tableDescriptor.setCompressionClass(DeflateCodec.class.getName());
+        tableDescriptor.setCompressionClass(DefaultCodec.class.getName());
       }
       if (tableDescriptor.getSimilarityClass() == null) {
         tableDescriptor.setSimilarityClass(FairSimilarity.class.getName());

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractIndexServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractIndexServer.java b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractIndexServer.java
index 03344af..b9e0e6b 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractIndexServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractIndexServer.java
@@ -18,119 +18,54 @@ package org.apache.blur.manager.indexserver;
  */
 import java.io.IOException;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.blur.manager.IndexServer;
 import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermDocs;
-
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
 
 public abstract class AbstractIndexServer implements IndexServer {
 
-  private Map<String, IndexCounts> _recordsTableCounts = new ConcurrentHashMap<String, IndexCounts>();
-  private Map<String, IndexCounts> _rowTableCounts = new ConcurrentHashMap<String, IndexCounts>();
-
-  private static class IndexCounts {
-    Map<String, IndexCount> counts = new ConcurrentHashMap<String, IndexCount>();
-  }
-
-  private static class IndexCount {
-    long version;
-    long count = -1;
-  }
-
   public long getRecordCount(String table) throws IOException {
-    IndexCounts indexCounts;
-    synchronized (_recordsTableCounts) {
-      indexCounts = _recordsTableCounts.get(table);
-      if (indexCounts == null) {
-        indexCounts = new IndexCounts();
-        _recordsTableCounts.put(table, indexCounts);
-      }
-    }
-    synchronized (indexCounts) {
-      long recordCount = 0;
-      Map<String, BlurIndex> indexes = getIndexes(table);
-      for (Map.Entry<String, BlurIndex> index : indexes.entrySet()) {
-        IndexReader indexReader = null;
-        try {
-          String shard = index.getKey();
-          IndexCount indexCount = indexCounts.counts.get(shard);
-          if (indexCount == null) {
-            indexCount = new IndexCount();
-            indexCounts.counts.put(shard, indexCount);
-          }
-          indexReader = index.getValue().getIndexReader();
-          if (!isValid(indexCount, indexReader)) {
-            indexCount.count = indexReader.numDocs();
-            indexCount.version = indexReader.getVersion();
-          }
-          recordCount += indexCount.count;
-        } finally {
-          if (indexReader != null) {
-            indexReader.decRef();
-          }
+    long recordCount = 0;
+    Map<String, BlurIndex> indexes = getIndexes(table);
+    for (Map.Entry<String, BlurIndex> index : indexes.entrySet()) {
+      IndexReader indexReader = null;
+      try {
+        indexReader = index.getValue().getIndexReader();
+        recordCount += indexReader.numDocs();
+      } finally {
+        if (indexReader != null) {
+          indexReader.decRef();
         }
       }
-      return recordCount;
     }
-  }
-
-  private boolean isValid(IndexCount indexCount, IndexReader indexReader) {
-    if (indexCount.version == indexReader.getVersion() && indexCount.count != -1l) {
-      return true;
-    }
-    return false;
+    return recordCount;
   }
 
   public long getRowCount(String table) throws IOException {
-    IndexCounts indexCounts;
-    synchronized (_rowTableCounts) {
-      indexCounts = _rowTableCounts.get(table);
-      if (indexCounts == null) {
-        indexCounts = new IndexCounts();
-        _rowTableCounts.put(table, indexCounts);
-      }
-    }
-    synchronized (indexCounts) {
-      long rowCount = 0;
-      Map<String, BlurIndex> indexes = getIndexes(table);
-      for (Map.Entry<String, BlurIndex> index : indexes.entrySet()) {
-        IndexReader indexReader = null;
-        try {
-          String shard = index.getKey();
-          IndexCount indexCount = indexCounts.counts.get(shard);
-          if (indexCount == null) {
-            indexCount = new IndexCount();
-            indexCounts.counts.put(shard, indexCount);
-          }
-          indexReader = index.getValue().getIndexReader();
-          if (!isValid(indexCount, indexReader)) {
-            indexCount.count = getRowCount(indexReader);
-            indexCount.version = indexReader.getVersion();
-          }
-          rowCount += indexCount.count;
-        } finally {
-          if (indexReader != null) {
-            indexReader.decRef();
-          }
+    long rowCount = 0;
+    Map<String, BlurIndex> indexes = getIndexes(table);
+    for (Map.Entry<String, BlurIndex> index : indexes.entrySet()) {
+      IndexReader indexReader = null;
+      try {
+        indexReader = index.getValue().getIndexReader();
+        rowCount += getRowCount(indexReader);
+      } finally {
+        if (indexReader != null) {
+          indexReader.decRef();
         }
       }
-      return rowCount;
     }
+    return rowCount;
   }
 
   private long getRowCount(IndexReader indexReader) throws IOException {
-    long rowCount = 0;
-    TermDocs termDocs = indexReader.termDocs(BlurConstants.PRIME_DOC_TERM);
-    while (termDocs.next()) {
-      if (!indexReader.isDeleted(termDocs.doc())) {
-        rowCount++;
-      }
-    }
-    termDocs.close();
-    return rowCount;
+    IndexSearcher searcher = new IndexSearcher(indexReader);
+    TopDocs topDocs = searcher.search(new TermQuery(BlurConstants.PRIME_DOC_TERM), 1);
+    return topDocs.totalHits;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmup.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmup.java b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmup.java
index f828229..25e76e3 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmup.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DefaultBlurIndexWarmup.java
@@ -17,24 +17,13 @@ package org.apache.blur.manager.indexserver;
  * limitations under the License.
  */
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.indexserver.DistributedIndexServer.ReleaseReader;
-import org.apache.blur.thrift.generated.ColumnPreCache;
 import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.utils.BlurConstants;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.WarmUpByFieldBounds;
-import org.apache.lucene.index.WarmUpByFieldBoundsStatus;
-import org.apache.lucene.util.ReaderUtil;
-
 
 public class DefaultBlurIndexWarmup extends BlurIndexWarmup {
 
@@ -42,48 +31,56 @@ public class DefaultBlurIndexWarmup extends BlurIndexWarmup {
 
   @Override
   public void warmBlurIndex(final TableDescriptor table, final String shard, IndexReader reader, AtomicBoolean isClosed, ReleaseReader releaseReader) throws IOException {
-    try {
-      ColumnPreCache columnPreCache = table.columnPreCache;
-      List<String> preCacheCols = null;
-      if (columnPreCache != null) {
-        preCacheCols = columnPreCache.preCacheCols;
-      }
-      if (preCacheCols == null) {
-        LOG.info("No pre cache defined, precache all fields.");
-        FieldInfos fieldInfos = ReaderUtil.getMergedFieldInfos(reader);
-        preCacheCols = new ArrayList<String>();
-        for (FieldInfo fieldInfo : fieldInfos) {
-          if (fieldInfo.isIndexed) {
-            preCacheCols.add(fieldInfo.name);
-          }
-        }
-        preCacheCols.remove(BlurConstants.ROW_ID);
-        preCacheCols.remove(BlurConstants.RECORD_ID);
-        preCacheCols.remove(BlurConstants.PRIME_DOC);
-        preCacheCols.remove(BlurConstants.SUPER);
-      }
+    LOG.warn("Warm up NOT supported yet.");
+    // Once the reader warm-up has been re-implemented, this code will change
+    // accordingly.
 
-      WarmUpByFieldBounds warmUpByFieldBounds = new WarmUpByFieldBounds();
-      WarmUpByFieldBoundsStatus status = new WarmUpByFieldBoundsStatus() {
-        @Override
-        public void complete(String name, Term start, Term end, long startPosition, long endPosition, long totalBytesRead, long nanoTime, AtomicBoolean isClosed) {
-          double bytesPerNano = totalBytesRead / (double) nanoTime;
-          double mBytesPerNano = bytesPerNano / 1024 / 1024;
-          double mBytesPerSecond = mBytesPerNano * 1000000000.0;
-          if (totalBytesRead > 0) {
-            LOG.info("Precached field [{0}] in table [{1}] shard [{2}] file [{3}], [{4}] bytes cached at [{5} MB/s]", start.field(), table.name, shard, name, totalBytesRead,
-                mBytesPerSecond);
-          }
-        }
-      };
-      if (preCacheCols != null) {
-        for (String field : preCacheCols) {
-          warmUpByFieldBounds.warmUpByField(isClosed, new Term(field), reader, status);
-        }
-      }
-    } finally {
-      releaseReader.release();
-    }
+    // try {
+    // ColumnPreCache columnPreCache = table.columnPreCache;
+    // List<String> preCacheCols = null;
+    // if (columnPreCache != null) {
+    // preCacheCols = columnPreCache.preCacheCols;
+    // }
+    // if (preCacheCols == null) {
+    // LOG.info("No pre cache defined, precache all fields.");
+    // FieldInfos fieldInfos = ReaderUtil.getMergedFieldInfos(reader);
+    // preCacheCols = new ArrayList<String>();
+    // for (FieldInfo fieldInfo : fieldInfos) {
+    // if (fieldInfo.isIndexed) {
+    // preCacheCols.add(fieldInfo.name);
+    // }
+    // }
+    // preCacheCols.remove(BlurConstants.ROW_ID);
+    // preCacheCols.remove(BlurConstants.RECORD_ID);
+    // preCacheCols.remove(BlurConstants.PRIME_DOC);
+    // preCacheCols.remove(BlurConstants.SUPER);
+    // }
+    //
+    // WarmUpByFieldBounds warmUpByFieldBounds = new WarmUpByFieldBounds();
+    // WarmUpByFieldBoundsStatus status = new WarmUpByFieldBoundsStatus() {
+    // @Override
+    // public void complete(String name, Term start, Term end, long
+    // startPosition, long endPosition, long totalBytesRead, long nanoTime,
+    // AtomicBoolean isClosed) {
+    // double bytesPerNano = totalBytesRead / (double) nanoTime;
+    // double mBytesPerNano = bytesPerNano / 1024 / 1024;
+    // double mBytesPerSecond = mBytesPerNano * 1000000000.0;
+    // if (totalBytesRead > 0) {
+    // LOG.info("Precached field [{0}] in table [{1}] shard [{2}] file [{3}], [{4}] bytes cached at [{5} MB/s]",
+    // start.field(), table.name, shard, name, totalBytesRead,
+    // mBytesPerSecond);
+    // }
+    // }
+    // };
+    // if (preCacheCols != null) {
+    // for (String field : preCacheCols) {
+    // warmUpByFieldBounds.warmUpByField(isClosed, new Term(field), reader,
+    // status);
+    // }
+    // }
+    // } finally {
+    // releaseReader.release();
+    // }
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index 65da0a0..6d0c8ea 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -20,7 +20,6 @@ import static org.apache.blur.utils.BlurConstants.SHARD_PREFIX;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -44,6 +43,7 @@ import org.apache.blur.concurrent.Executors;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.search.FairSimilarity;
+import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
 import org.apache.blur.manager.BlurFilterCache;
 import org.apache.blur.manager.clusterstatus.ClusterStatus;
 import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
@@ -52,13 +52,11 @@ import org.apache.blur.manager.writer.BlurIndexCloser;
 import org.apache.blur.manager.writer.BlurIndexReader;
 import org.apache.blur.manager.writer.BlurIndexRefresher;
 import org.apache.blur.manager.writer.BlurNRTIndex;
-import org.apache.blur.manager.writer.DirectoryReferenceFileGC;
 import org.apache.blur.metrics.BlurMetrics;
 import org.apache.blur.store.blockcache.BlockDirectory;
 import org.apache.blur.store.blockcache.Cache;
-import org.apache.blur.store.compressed.CompressedFieldDataDirectory;
+import org.apache.blur.store.hdfs.BlurLockFactory;
 import org.apache.blur.store.hdfs.HdfsDirectory;
-import org.apache.blur.store.lock.BlurLockFactory;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurUtil;
@@ -71,15 +69,10 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexDeletionPolicy;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermPositions;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.ReaderUtil;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.Ids;
@@ -269,18 +262,19 @@ public class DistributedIndexServer extends AbstractIndexServer {
       }
 
       private void updateMetrics(BlurMetrics blurMetrics, Map<String, BlurIndex> indexes, AtomicLong segmentCount, AtomicLong indexMemoryUsage) throws IOException {
-        for (BlurIndex index : indexes.values()) {
-          IndexReader reader = index.getIndexReader();
-          try {
-            IndexReader[] readers = reader.getSequentialSubReaders();
-            if (readers != null) {
-              segmentCount.addAndGet(readers.length);
-            }
-            indexMemoryUsage.addAndGet(BlurUtil.getMemoryUsage(reader));
-          } finally {
-            reader.decRef();
-          }
-        }
+        // @TODO not sure how to do this yet
+//        for (BlurIndex index : indexes.values()) {
+//          IndexReader reader = index.getIndexReader();
+//          try {
+//            IndexReader[] readers = reader.getSequentialSubReaders();
+//            if (readers != null) {
+//              segmentCount.addAndGet(readers.length);
+//            }
+//            indexMemoryUsage.addAndGet(BlurUtil.getMemoryUsage(reader));
+//          } finally {
+//            reader.decRef();
+//          }
+//        }
       }
     }, _delay, _delay);
   }
@@ -478,20 +472,22 @@ public class DistributedIndexServer extends AbstractIndexServer {
 
     BlurLockFactory lockFactory = new BlurLockFactory(_configuration, hdfsDirPath, _nodeName, BlurConstants.getPid());
 
-    Directory directory = new HdfsDirectory(hdfsDirPath);
+    Directory directory = new HdfsDirectory(_configuration, hdfsDirPath);
     directory.setLockFactory(lockFactory);
 
     TableDescriptor descriptor = _clusterStatus.getTableDescriptor(true, _cluster, table);
     String compressionClass = descriptor.compressionClass;
     int compressionBlockSize = descriptor.compressionBlockSize;
     if (compressionClass != null) {
-      CompressionCodec compressionCodec;
-      try {
-        compressionCodec = BlurUtil.getInstance(compressionClass, CompressionCodec.class);
-        directory = new CompressedFieldDataDirectory(directory, compressionCodec, compressionBlockSize);
-      } catch (Exception e) {
-        throw new IOException(e);
-      }
+//      throw new RuntimeException("Not supported yet");
+      LOG.error("Not supported yet");
+//      CompressionCodec compressionCodec;
+//      try {
+//        compressionCodec = BlurUtil.getInstance(compressionClass, CompressionCodec.class);
+//        directory = new CompressedFieldDataDirectory(directory, compressionCodec, compressionBlockSize);
+//      } catch (Exception e) {
+//        throw new IOException(e);
+//      }
     }
 
     Directory dir;
@@ -553,35 +549,38 @@ public class DistributedIndexServer extends AbstractIndexServer {
   }
 
   private void warmUpAllSegments(IndexReader reader) throws IOException {
-    IndexReader[] indexReaders = reader.getSequentialSubReaders();
-    if (indexReaders != null) {
-      for (IndexReader r : indexReaders) {
-        warmUpAllSegments(r);
-      }
-    }
-    int maxDoc = reader.maxDoc();
-    int numDocs = reader.numDocs();
-    FieldInfos fieldInfos = ReaderUtil.getMergedFieldInfos(reader);
-    Collection<String> fieldNames = new ArrayList<String>();
-    for (FieldInfo fieldInfo : fieldInfos) {
-      if (fieldInfo.isIndexed) {
-        fieldNames.add(fieldInfo.name);
-      }
-    }
-    int primeDocCount = reader.docFreq(BlurConstants.PRIME_DOC_TERM);
-    TermDocs termDocs = reader.termDocs(BlurConstants.PRIME_DOC_TERM);
-    termDocs.next();
-    termDocs.close();
-
-    TermPositions termPositions = reader.termPositions(BlurConstants.PRIME_DOC_TERM);
-    if (termPositions.next()) {
-      if (termPositions.freq() > 0) {
-        termPositions.nextPosition();
-      }
-    }
-    termPositions.close();
-    LOG.info("Warmup of indexreader [" + reader + "] complete, maxDocs [" + maxDoc + "], numDocs [" + numDocs + "], primeDocumentCount [" + primeDocCount + "], fieldCount ["
-        + fieldNames.size() + "]");
+    LOG.warn("Warm up NOT supported yet.");
+    //Once the reader warm-up has been re-implemented, this code will change accordingly.
+    
+//    IndexReader[] indexReaders = reader.getSequentialSubReaders();
+//    if (indexReaders != null) {
+//      for (IndexReader r : indexReaders) {
+//        warmUpAllSegments(r);
+//      }
+//    }
+//    int maxDoc = reader.maxDoc();
+//    int numDocs = reader.numDocs();
+//    FieldInfos fieldInfos = ReaderUtil.getMergedFieldInfos(reader);
+//    Collection<String> fieldNames = new ArrayList<String>();
+//    for (FieldInfo fieldInfo : fieldInfos) {
+//      if (fieldInfo.isIndexed) {
+//        fieldNames.add(fieldInfo.name);
+//      }
+//    }
+//    int primeDocCount = reader.docFreq(BlurConstants.PRIME_DOC_TERM);
+//    TermDocs termDocs = reader.termDocs(BlurConstants.PRIME_DOC_TERM);
+//    termDocs.next();
+//    termDocs.close();
+//
+//    TermPositions termPositions = reader.termPositions(BlurConstants.PRIME_DOC_TERM);
+//    if (termPositions.next()) {
+//      if (termPositions.freq() > 0) {
+//        termPositions.nextPosition();
+//      }
+//    }
+//    termPositions.close();
+//    LOG.info("Warmup of indexreader [" + reader + "] complete, maxDocs [" + maxDoc + "], numDocs [" + numDocs + "], primeDocumentCount [" + primeDocCount + "], fieldCount ["
+//        + fieldNames.size() + "]");
   }
 
   private synchronized Map<String, BlurIndex> openMissingShards(final String table, Set<String> shardsToServe, final Map<String, BlurIndex> tableIndexes) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
index d5ee79f..041eb2a 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
@@ -16,7 +16,7 @@ package org.apache.blur.manager.indexserver;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import static org.apache.blur.lucene.LuceneConstant.LUCENE_VERSION;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 
 import java.io.File;
 import java.io.IOException;
@@ -38,21 +38,21 @@ import org.apache.blur.lucene.search.FairSimilarity;
 import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.manager.writer.BlurIndexCloser;
 import org.apache.blur.manager.writer.BlurNRTIndex;
-import org.apache.blur.store.compressed.CompressedFieldDataDirectory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.lucene.analysis.KeywordAnalyzer;
+import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.util.CharArraySet;
 import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MMapDirectory;
 
-
 public class LocalIndexServer extends AbstractIndexServer {
 
   private final static Log LOG = LogFactory.getLog(LocalIndexServer.class);
@@ -61,7 +61,7 @@ public class LocalIndexServer extends AbstractIndexServer {
   private File _localDir;
   private BlurIndexCloser _closer;
   private int _blockSize = 65536;
-  private CompressionCodec _compression = CompressedFieldDataDirectory.DEFAULT_COMPRESSION;
+  private CompressionCodec _compression = new DefaultCodec();
   private Path _walPath;
   private Configuration _configuration = new Configuration();
 
@@ -75,7 +75,7 @@ public class LocalIndexServer extends AbstractIndexServer {
 
   @Override
   public BlurAnalyzer getAnalyzer(String table) {
-    return new BlurAnalyzer(new StandardAnalyzer(LUCENE_VERSION, new HashSet<String>()));
+    return new BlurAnalyzer(new StandardAnalyzer(LUCENE_VERSION, new CharArraySet(LUCENE_VERSION, new HashSet<String>(), false)));
   }
 
   @Override
@@ -130,7 +130,7 @@ public class LocalIndexServer extends AbstractIndexServer {
       for (File f : tableFile.listFiles()) {
         if (f.isDirectory()) {
           MMapDirectory directory = new MMapDirectory(f);
-          if (!IndexReader.indexExists(directory)) {
+          if (!DirectoryReader.indexExists(directory)) {
             new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer())).close();
           }
           String shardName = f.getName();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8a0b462f/src/blur-core/src/main/java/org/apache/blur/manager/writer/AbstractBlurIndex.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/writer/AbstractBlurIndex.java b/src/blur-core/src/main/java/org/apache/blur/manager/writer/AbstractBlurIndex.java
index 5138bd2..b758f11 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/writer/AbstractBlurIndex.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/writer/AbstractBlurIndex.java
@@ -16,7 +16,7 @@ package org.apache.blur.manager.writer;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import static org.apache.blur.lucene.LuceneConstant.LUCENE_VERSION;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 
 import java.io.IOException;
 import java.util.concurrent.Callable;
@@ -25,12 +25,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexDeletionPolicy;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
 import org.apache.lucene.index.TieredMergePolicy;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.Directory;
 
 public abstract class AbstractBlurIndex extends BlurIndex {
@@ -39,7 +40,7 @@ public abstract class AbstractBlurIndex extends BlurIndex {
   private BlurIndexCloser _closer;
   private Directory _directory;
   private IndexDeletionPolicy _indexDeletionPolicy = new KeepOnlyLastCommitDeletionPolicy();
-  private AtomicReference<IndexReader> _indexReaderRef = new AtomicReference<IndexReader>();
+  private AtomicReference<DirectoryReader> _indexReaderRef = new AtomicReference<DirectoryReader>();
   private AtomicBoolean _isClosed = new AtomicBoolean(false);
   private AtomicBoolean _open = new AtomicBoolean();
   private BlurIndexRefresher _refresher;
@@ -58,7 +59,7 @@ public abstract class AbstractBlurIndex extends BlurIndex {
     return conf;
   }
 
-  protected void initIndexReader(IndexReader reader) throws IOException {
+  protected void initIndexReader(DirectoryReader reader) throws IOException {
     _indexReaderRef.set(reader);
     _refresher.register(this);
   }
@@ -68,12 +69,9 @@ public abstract class AbstractBlurIndex extends BlurIndex {
     if (!_open.get()) {
       return;
     }
-    IndexReader oldReader = _indexReaderRef.get();
-    if (oldReader.isCurrent()) {
-      return;
-    }
-    IndexReader reader = IndexReader.openIfChanged(oldReader);
-    if (reader != null && oldReader != reader) {
+    DirectoryReader oldReader = _indexReaderRef.get();
+    DirectoryReader reader = DirectoryReader.openIfChanged(oldReader);
+    if (reader != null) {
       _indexReaderRef.set(reader);
       _closer.close(oldReader);
     }


Mime
View raw message