incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [09/39] git commit: Still broken, but needed to check in the changes I have made thus far.
Date Mon, 12 Aug 2013 15:48:48 GMT
Still broken, but needed to check in the changes I have made thus far.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/4ebe74ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/4ebe74ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/4ebe74ef

Branch: refs/heads/master
Commit: 4ebe74ef2e489d8a360220e0d2752c682042ab22
Parents: 2498ff1
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Jul 2 19:17:02 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Jul 2 19:17:02 2013 -0400

----------------------------------------------------------------------
 .../org/apache/blur/manager/IndexManager.java   |  58 +-
 .../org/apache/blur/manager/IndexServer.java    |  10 -
 .../apache/blur/manager/QueryParserUtil.java    |  10 +-
 .../clusterstatus/ZookeeperClusterStatus.java   |   1 -
 .../indexserver/DistributedIndexServer.java     |  15 -
 .../manager/indexserver/LocalIndexServer.java   |   6 -
 .../blur/manager/writer/AbstractBlurIndex.java  | 163 ------
 .../blur/manager/writer/BlurIndexReader.java    |   3 +-
 .../blur/manager/writer/BlurNRTIndex.java       |   6 +-
 .../manager/writer/TransactionRecorder.java     |  99 ++--
 .../org/apache/blur/server/TableContext.java    |  10 +-
 .../org/apache/blur/utils/HighlightHelper.java  |  13 +-
 .../blur/mapreduce/lib/BlurOutputFormat.java    |   5 +-
 .../apache/blur/analysis/BaseFieldManager.java  | 142 ++++-
 .../org/apache/blur/analysis/BlurAnalyzer.java  | 531 -------------------
 .../org/apache/blur/analysis/FieldManager.java  |  82 ++-
 .../blur/analysis/FieldTypeDefinition.java      |  11 +-
 .../type/DoubleFieldTypeDefinition.java         |  27 +-
 .../analysis/type/FloatFieldTypeDefinition.java |  25 +-
 .../analysis/type/IntFieldTypeDefinition.java   |  26 +-
 .../analysis/type/LongFieldTypeDefinition.java  |  25 +-
 .../type/NumericFieldTypeDefinition.java        |  65 +++
 .../type/StoredFieldTypeDefinition.java         |  20 +
 .../type/StringFieldTypeDefinition.java         |  20 +
 .../analysis/type/TextFieldTypeDefinition.java  |  20 +
 .../blur/lucene/search/BlurQueryParser.java     |  33 +-
 .../apache/blur/lucene/search/SuperParser.java  |   8 +-
 .../blur/analysis/BaseFiledManagerTest.java     |   3 +-
 .../apache/blur/analysis/BlurAnalyzerTest.java  | 256 +++++----
 .../blur/lucene/search/SuperParserTest.java     |  52 +-
 30 files changed, 671 insertions(+), 1074 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 368aa75..5996cc8 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -45,7 +45,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLongArray;
 
-import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.index.ExitableReader;
 import org.apache.blur.index.ExitableReader.ExitingReaderException;
@@ -223,11 +223,13 @@ public class IndexManager {
         searcher = index.getIndexReader();
         usedCache = false;
       }
-      BlurAnalyzer analyzer = _indexServer.getAnalyzer(table);
+      
+      TableContext tableContext = getTableContext(table);
+      FieldManager fieldManager = tableContext.getFieldManager();
+      
+      Query highlightQuery = getHighlightQuery(selector, table, fieldManager);
 
-      Query highlightQuery = getHighlightQuery(selector, table, analyzer);
-
-      fetchRow(searcher.getIndexReader(), table, selector, fetchResult, highlightQuery, analyzer);
+      fetchRow(searcher.getIndexReader(), table, selector, fetchResult, highlightQuery, fieldManager);
       if (fetchResult.rowResult != null) {
         if (fetchResult.rowResult.row != null && fetchResult.rowResult.row.records != null) {
           _recordsMeter.mark(fetchResult.rowResult.row.records.size());
@@ -253,7 +255,7 @@ public class IndexManager {
     }
   }
 
-  private Query getHighlightQuery(Selector selector, String table, BlurAnalyzer analyzer) throws ParseException,
+  private Query getHighlightQuery(Selector selector, String table, FieldManager fieldManager) throws ParseException,
       BlurException {
     HighlightOptions highlightOptions = selector.getHighlightOptions();
     if (highlightOptions == null) {
@@ -265,11 +267,11 @@ public class IndexManager {
     }
 
     TableContext context = getTableContext(table);
-    Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter, false, analyzer, _filterCache,
+    Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter, false, fieldManager, _filterCache,
         context);
-    Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter, true, analyzer, _filterCache,
+    Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter, true, fieldManager, _filterCache,
         context);
-    return QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn, analyzer, postFilter, preFilter,
+    return QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn, fieldManager, postFilter, preFilter,
         getScoreType(simpleQuery.type), context);
   }
 
@@ -372,18 +374,18 @@ public class IndexManager {
         throw new BException(e.getMessage(), e);
       }
       ShardServerContext shardServerContext = ShardServerContext.getShardServerContext();
-      BlurAnalyzer analyzer = _indexServer.getAnalyzer(table);
       ParallelCall<Entry<String, BlurIndex>, BlurResultIterable> call;
       TableContext context = getTableContext(table);
+      FieldManager fieldManager = context.getFieldManager();
       if (isSimpleQuery(blurQuery)) {
         SimpleQuery simpleQuery = blurQuery.simpleQuery;
-        Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter, false, analyzer,
+        Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter, false, fieldManager,
             _filterCache, context);
-        Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter, true, analyzer,
+        Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter, true, fieldManager,
             _filterCache, context);
-        Query userQuery = QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn, analyzer,
+        Query userQuery = QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn, fieldManager,
             postFilter, preFilter, getScoreType(simpleQuery.type), context);
-        Query facetedQuery = getFacetedQuery(blurQuery, userQuery, facetedCounts, analyzer, context, postFilter,
+        Query facetedQuery = getFacetedQuery(blurQuery, userQuery, facetedCounts, fieldManager, context, postFilter,
             preFilter);
         call = new SimpleQueryParallelCall(running, table, status, _indexServer, facetedQuery, blurQuery.selector,
             _queriesInternalMeter, shardServerContext, runSlow);
@@ -396,7 +398,7 @@ public class IndexManager {
         } else {
           userQuery = query;
         }
-        Query facetedQuery = getFacetedQuery(blurQuery, userQuery, facetedCounts, analyzer, context, null, null);
+        Query facetedQuery = getFacetedQuery(blurQuery, userQuery, facetedCounts, fieldManager, context, null, null);
         call = new SimpleQueryParallelCall(running, table, status, _indexServer, facetedQuery, blurQuery.selector,
             _queriesInternalMeter, shardServerContext, runSlow);
       }
@@ -432,12 +434,12 @@ public class IndexManager {
 
   public String parseQuery(String table, SimpleQuery simpleQuery) throws ParseException, BlurException {
     TableContext context = getTableContext(table);
-    BlurAnalyzer analyzer = _indexServer.getAnalyzer(table);
-    Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter, false, analyzer, _filterCache,
+    FieldManager fieldManager = context.getFieldManager();
+    Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter, false, fieldManager, _filterCache,
         context);
-    Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter, true, analyzer, _filterCache,
+    Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter, true, fieldManager, _filterCache,
         context);
-    Query userQuery = QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn, analyzer, postFilter,
+    Query userQuery = QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn, fieldManager, postFilter,
         preFilter, getScoreType(simpleQuery.type), context);
     return userQuery.toString();
   }
@@ -461,21 +463,21 @@ public class IndexManager {
     return false;
   }
 
-  private Query getFacetedQuery(BlurQuery blurQuery, Query userQuery, AtomicLongArray counts, BlurAnalyzer analyzer,
+  private Query getFacetedQuery(BlurQuery blurQuery, Query userQuery, AtomicLongArray counts, FieldManager fieldManager,
       TableContext context, Filter postFilter, Filter preFilter) throws ParseException {
     if (blurQuery.facets == null) {
       return userQuery;
     }
-    return new FacetQuery(userQuery, getFacetQueries(blurQuery, analyzer, context, postFilter, preFilter), counts);
+    return new FacetQuery(userQuery, getFacetQueries(blurQuery, fieldManager, context, postFilter, preFilter), counts);
   }
 
-  private Query[] getFacetQueries(BlurQuery blurQuery, BlurAnalyzer analyzer, TableContext context, Filter postFilter,
+  private Query[] getFacetQueries(BlurQuery blurQuery, FieldManager fieldManager, TableContext context, Filter postFilter,
       Filter preFilter) throws ParseException {
     int size = blurQuery.facets.size();
     Query[] queries = new Query[size];
     for (int i = 0; i < size; i++) {
       queries[i] = QueryParserUtil.parseQuery(blurQuery.facets.get(i).queryStr, blurQuery.simpleQuery.superQueryOn,
-          analyzer, postFilter, preFilter, ScoreType.CONSTANT, context);
+          fieldManager, postFilter, preFilter, ScoreType.CONSTANT, context);
     }
     return queries;
   }
@@ -509,7 +511,7 @@ public class IndexManager {
   }
 
   public static void fetchRow(IndexReader reader, String table, Selector selector, FetchResult fetchResult,
-      Query highlightQuery, BlurAnalyzer analyzer) throws CorruptIndexException, IOException {
+      Query highlightQuery, FieldManager fieldManager) throws CorruptIndexException, IOException {
     fetchResult.table = table;
     String locationId = selector.locationId;
     int lastSlash = locationId.lastIndexOf('/');
@@ -538,12 +540,12 @@ public class IndexManager {
         fetchResult.deleted = false;
         reader.document(docId, fieldVisitor);
         Document document = fieldVisitor.getDocument();
-        if (highlightQuery != null && analyzer != null) {
+        if (highlightQuery != null && fieldManager != null) {
           HighlightOptions highlightOptions = selector.getHighlightOptions();
           String preTag = highlightOptions.getPreTag();
           String postTag = highlightOptions.getPostTag();
           try {
-            document = HighlightHelper.highlight(docId, document, highlightQuery, analyzer, reader, preTag, postTag);
+            document = HighlightHelper.highlight(docId, document, highlightQuery, fieldManager, reader, preTag, postTag);
           } catch (InvalidTokenOffsetsException e) {
             LOG.error("Unknown error while tring to highlight", e);
           }
@@ -568,11 +570,11 @@ public class IndexManager {
           fetchResult.rowResult.row = new Row(rowId, null, recordCount);
         } else {
           List<Document> docs;
-          if (highlightQuery != null && analyzer != null) {
+          if (highlightQuery != null && fieldManager != null) {
             HighlightOptions highlightOptions = selector.getHighlightOptions();
             String preTag = highlightOptions.getPreTag();
             String postTag = highlightOptions.getPostTag();
-            docs = HighlightHelper.highlightDocuments(reader, term, fieldVisitor, selector, highlightQuery, analyzer,
+            docs = HighlightHelper.highlightDocuments(reader, term, fieldVisitor, selector, highlightQuery, fieldManager,
                 preTag, postTag);
           } else {
             docs = BlurUtil.fetchDocuments(reader, term, fieldVisitor, selector);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
index 85bd1b7..7147c53 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
@@ -21,7 +21,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.SortedSet;
 
-import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.thrift.generated.ShardState;
 import org.apache.lucene.search.similarities.Similarity;
@@ -92,15 +91,6 @@ public interface IndexServer {
   TABLE_STATUS getTableStatus(String table);
 
   /**
-   * Gets the analyzer for the table.
-   * 
-   * @param table
-   *          the table name.
-   * @return the analyzer for lucene.
-   */
-  BlurAnalyzer getAnalyzer(String table);
-
-  /**
    * Gets the current nodes name.
    * 
    * @return

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java b/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
index 4d12067..57a43a7 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/QueryParserUtil.java
@@ -18,7 +18,7 @@ package org.apache.blur.manager;
  */
 import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 
-import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.lucene.search.SuperParser;
 import org.apache.blur.server.TableContext;
 import org.apache.blur.thrift.generated.BlurException;
@@ -31,15 +31,15 @@ import org.apache.lucene.search.QueryWrapperFilter;
 
 public class QueryParserUtil {
 
-  public static Query parseQuery(String query, boolean superQueryOn, BlurAnalyzer analyzer, Filter postFilter, Filter preFilter, ScoreType scoreType, TableContext tableContext) throws ParseException   {
-    Query result = new SuperParser(LUCENE_VERSION, analyzer,superQueryOn, preFilter,  scoreType, tableContext.getDefaultPrimeDocTerm()).parse(query);
+  public static Query parseQuery(String query, boolean superQueryOn, FieldManager fieldManager, Filter postFilter, Filter preFilter, ScoreType scoreType, TableContext tableContext) throws ParseException   {
+    Query result = new SuperParser(LUCENE_VERSION, fieldManager,superQueryOn, preFilter,  scoreType, tableContext.getDefaultPrimeDocTerm()).parse(query);
     if (postFilter == null) {
       return result;
     }
     return new FilteredQuery(result, postFilter);
   }
 
-  public static Filter parseFilter(String table, String filterStr, boolean superQueryOn, BlurAnalyzer analyzer, BlurFilterCache filterCache, TableContext tableContext) throws ParseException, BlurException {
+  public static Filter parseFilter(String table, String filterStr, boolean superQueryOn, FieldManager fieldManager, BlurFilterCache filterCache, TableContext tableContext) throws ParseException, BlurException {
     if (filterStr == null) {
       return null;
     }
@@ -53,7 +53,7 @@ public class QueryParserUtil {
       if (filter != null) {
         return filter;
       }
-      filter = new QueryWrapperFilter(new SuperParser(LUCENE_VERSION, analyzer, superQueryOn, null,  ScoreType.CONSTANT, tableContext.getDefaultPrimeDocTerm()).parse(filterStr));
+      filter = new QueryWrapperFilter(new SuperParser(LUCENE_VERSION, fieldManager, superQueryOn, null,  ScoreType.CONSTANT, tableContext.getDefaultPrimeDocTerm()).parse(filterStr));
       if (superQueryOn) {
         filter = filterCache.storePostFilter(table, filterStr, filter);
       } else {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
index b6bf61a..1fc6a43 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
@@ -32,7 +32,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
-import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.search.FairSimilarity;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index f0f449e..9cd4803 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -44,7 +44,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
@@ -98,7 +97,6 @@ public class DistributedIndexServer extends AbstractIndexServer {
   private static final long _delay = TimeUnit.SECONDS.toMillis(10);
   private static final long CHECK_PERIOD = TimeUnit.SECONDS.toMillis(60);
 
-  private Map<String, BlurAnalyzer> _tableAnalyzers = new ConcurrentHashMap<String, BlurAnalyzer>();
   private Map<String, TableDescriptor> _tableDescriptors = new ConcurrentHashMap<String, TableDescriptor>();
   private Map<String, Similarity> _tableSimilarity = new ConcurrentHashMap<String, Similarity>();
   private Map<String, DistributedLayoutManager> _layoutManagers = new ConcurrentHashMap<String, DistributedLayoutManager>();
@@ -318,7 +316,6 @@ public class DistributedIndexServer extends AbstractIndexServer {
       }
 
       private void cleanup() {
-        clearMapOfOldTables(_tableAnalyzers);
         clearMapOfOldTables(_tableDescriptors);
         clearMapOfOldTables(_layoutManagers);
         clearMapOfOldTables(_layoutCache);
@@ -423,18 +420,6 @@ public class DistributedIndexServer extends AbstractIndexServer {
   }
 
   @Override
-  public BlurAnalyzer getAnalyzer(String table) {
-    checkTable(table);
-    BlurAnalyzer blurAnalyzer = _tableAnalyzers.get(table);
-    if (blurAnalyzer == null) {
-      TableDescriptor descriptor = getTableDescriptor(table);
-      blurAnalyzer = new BlurAnalyzer(descriptor.analyzerDefinition);
-      _tableAnalyzers.put(table, blurAnalyzer);
-    }
-    return blurAnalyzer;
-  }
-
-  @Override
   public SortedSet<String> getShardListCurrentServerOnly(String table) throws IOException {
     return new TreeSet<String>(getShardsToServe(table));
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
index 453afcd..2ffa20a 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
@@ -32,7 +32,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
-import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.search.FairSimilarity;
@@ -101,11 +100,6 @@ public class LocalIndexServer extends AbstractIndexServer {
   }
 
   @Override
-  public BlurAnalyzer getAnalyzer(String table) {
-    return _tableContext.getAnalyzer();
-  }
-
-  @Override
   public SortedSet<String> getShardListCurrentServerOnly(String table) throws IOException {
     Map<String, BlurIndex> tableMap = _readersMap.get(table);
     Set<String> shardsSet;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/manager/writer/AbstractBlurIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/AbstractBlurIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/AbstractBlurIndex.java
deleted file mode 100644
index 9808803..0000000
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/AbstractBlurIndex.java
+++ /dev/null
@@ -1,163 +0,0 @@
-package org.apache.blur.manager.writer;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.blur.analysis.BlurAnalyzer;
-import org.apache.blur.server.IndexSearcherClosable;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexDeletionPolicy;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
-import org.apache.lucene.index.TieredMergePolicy;
-import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.store.Directory;
-
-public abstract class AbstractBlurIndex extends BlurIndex {
-
-  private BlurAnalyzer _analyzer;
-  private BlurIndexCloser _closer;
-  private Directory _directory;
-  private IndexDeletionPolicy _indexDeletionPolicy = new KeepOnlyLastCommitDeletionPolicy();
-  private AtomicReference<DirectoryReader> _indexReaderRef = new AtomicReference<DirectoryReader>();
-  private AtomicBoolean _isClosed = new AtomicBoolean(false);
-  private AtomicBoolean _open = new AtomicBoolean();
-  private BlurIndexRefresher _refresher;
-  private String _shard;
-  private Similarity _similarity;
-  private String _table;
-
-  protected IndexWriterConfig initIndexWriterConfig() {
-    IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _analyzer);
-    conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
-    conf.setIndexDeletionPolicy(_indexDeletionPolicy);
-    conf.setSimilarity(_similarity);
-    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
-    mergePolicy.setUseCompoundFile(false);
-    _open.set(true);
-    return conf;
-  }
-
-  protected void initIndexReader(DirectoryReader reader) throws IOException {
-    _indexReaderRef.set(reader);
-    _refresher.register(this);
-  }
-
-  @Override
-  public void refresh() throws IOException {
-    if (!_open.get()) {
-      return;
-    }
-    DirectoryReader oldReader = _indexReaderRef.get();
-    DirectoryReader reader = DirectoryReader.openIfChanged(oldReader);
-    if (reader != null) {
-      _indexReaderRef.set(reader);
-      _closer.close(oldReader);
-    }
-  }
-
-  @Override
-  public IndexSearcherClosable getIndexReader() throws IOException {
-    throw new RuntimeException("not impl");
-//    IndexReader indexReader = _indexReaderRef.get();
-//    indexReader.incRef();
-//    return indexReader;
-  }
-
-  @Override
-  public void close() throws IOException {
-    close(null);
-    _directory.close();
-  }
-
-  public void close(Callable<Void> innerClose) throws IOException {
-    _open.set(false);
-    _refresher.unregister(this);
-    if (innerClose != null) {
-      try {
-        innerClose.call();
-      } catch (Exception e) {
-        throw new IOException(e);
-      }
-    }
-    _isClosed.set(true);
-  }
-
-  @Override
-  public AtomicBoolean isClosed() {
-    return _isClosed;
-  }
-
-  public void setAnalyzer(BlurAnalyzer analyzer) {
-    _analyzer = analyzer;
-  }
-
-  public void setCloser(BlurIndexCloser closer) {
-    _closer = closer;
-  }
-
-  public void setDirectory(Directory directory) {
-    _directory = directory;
-  }
-
-  public void setIndexDeletionPolicy(IndexDeletionPolicy indexDeletionPolicy) {
-    _indexDeletionPolicy = indexDeletionPolicy;
-  }
-
-  public void setRefresher(BlurIndexRefresher refresher) {
-    _refresher = refresher;
-  }
-
-  public void setShard(String shard) {
-    this._shard = shard;
-  }
-
-  public void setSimilarity(Similarity similarity) {
-    _similarity = similarity;
-  }
-
-  public void setTable(String table) {
-    this._table = table;
-  }
-
-  protected BlurAnalyzer getAnalyzer() {
-    return _analyzer;
-  }
-
-  protected Directory getDirectory() {
-    return _directory;
-  }
-
-  protected String getShard() {
-    return _shard;
-  }
-
-  protected String getTable() {
-    return _table;
-  }
-
-  protected boolean isOpen() {
-    return _open.get();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
index e2e3bf6..aaeb6dd 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
@@ -30,6 +30,7 @@ import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
 import org.apache.blur.thrift.generated.Row;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.index.BlurIndexWriter;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -62,7 +63,7 @@ public class BlurIndexReader extends BlurIndex {
     if (!DirectoryReader.indexExists(directory)) {
       LOG.info("Creating an empty index");
       // if the directory is empty then create an empty index.
-      IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _tableContext.getAnalyzer());
+      IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
       conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
       new BlurIndexWriter(directory, conf).close();
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index c53af2e..1c36bc4 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.index.ExitableReader;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
@@ -40,6 +41,7 @@ import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
 import org.apache.blur.thrift.generated.Record;
 import org.apache.blur.thrift.generated.Row;
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.BlurIndexWriter;
 import org.apache.lucene.index.BlurIndexWriter.LockOwnerException;
 import org.apache.lucene.index.CorruptIndexException;
@@ -81,7 +83,9 @@ public class BlurNRTIndex extends BlurIndex {
     _directory = directory;
     _shardContext = shardContext;
 
-    IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _tableContext.getAnalyzer());
+    FieldManager fieldManager = _tableContext.getFieldManager();
+    Analyzer analyzer = fieldManager.getAnalyzerForIndex();
+    IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, analyzer);
     conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
     conf.setSimilarity(_tableContext.getSimilarity());
     conf.setIndexDeletionPolicy(_tableContext.getIndexDeletionPolicy());

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
index f8adfac..ab9bfc6 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
@@ -17,7 +17,6 @@ package org.apache.blur.manager.writer;
  * limitations under the License.
  */
 import static org.apache.blur.utils.BlurConstants.SEP;
-import static org.apache.blur.utils.BlurConstants.SUPER;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -28,7 +27,6 @@ import java.io.EOFException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.TimeUnit;
@@ -36,7 +34,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
-import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.server.ShardContext;
@@ -52,7 +50,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.record.Utils;
-import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.FieldType;
@@ -112,16 +109,16 @@ public class TransactionRecorder extends TimerTask implements Closeable {
 
   private final Path _walPath;
   private final Configuration _configuration;
-  private final BlurAnalyzer _analyzer;
   private final FileSystem _fileSystem;
   private final Timer _timer;
   private final String _table;
   private final String _shard;
+  private final FieldManager _fieldManager;
 
   public TransactionRecorder(ShardContext shardContext) throws IOException {
     TableContext tableContext = shardContext.getTableContext();
     _configuration = tableContext.getConfiguration();
-    _analyzer = tableContext.getAnalyzer();
+    _fieldManager = tableContext.getFieldManager();
     _walPath = shardContext.getWalShardPath();
     _fileSystem = _walPath.getFileSystem(_configuration);
     _timeBetweenSyncsNanos = tableContext.getTimeBetweenWALSyncsNanos();
@@ -165,7 +162,7 @@ public class TransactionRecorder extends TimerTask implements Closeable {
       switch (lookup) {
       case ROW:
         Row row = readRow(dataInputStream);
-        writer.updateDocuments(createRowId(row.id), getDocs(row, _analyzer));
+        writer.updateDocuments(createRowId(row.id), getDocs(row, _fieldManager));
         updateCount++;
         continue;
       case DELETE:
@@ -329,7 +326,7 @@ public class TransactionRecorder extends TimerTask implements Closeable {
       }
     }
     Term term = createRowId(row.id);
-    List<Document> docs = getDocs(row, _analyzer);
+    List<List<Field>> docs = getDocs(row, _fieldManager);
     return writer.updateDocuments(term, docs);
   }
 
@@ -359,30 +356,32 @@ public class TransactionRecorder extends TimerTask implements Closeable {
     }
   }
 
-  public static List<Document> getDocs(Row row, BlurAnalyzer analyzer) {
+  public static List<List<Field>> getDocs(Row row, FieldManager fieldManager) {
     List<Record> records = row.records;
     int size = records.size();
     final String rowId = row.id;
-    List<Document> docs = new ArrayList<Document>(size);
+    List<List<Field>> docs = new ArrayList<List<Field>>(size);
     for (int i = 0; i < size; i++) {
-      Document document = convert(rowId, records.get(i), analyzer);
+      Record record = records.get(i);
+      BlurUtil.validateRowIdAndRecord(rowId, record);
+      List<Field> fields = fieldManager.getFields(rowId, record);
       if (i == 0) {
-        document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
+        fields.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
       }
-      docs.add(document);
+      docs.add(fields);
     }
     return docs;
   }
 
-  public static Document convert(String rowId, Record record, BlurAnalyzer analyzer) {
-    BlurUtil.validateRowIdAndRecord(rowId, record);
-    Document document = new Document();
-    document.add(new Field(BlurConstants.ROW_ID, rowId, ID_TYPE));
-    document.add(new Field(BlurConstants.RECORD_ID, record.recordId, ID_TYPE));
-    document.add(new Field(BlurConstants.FAMILY, record.family, ID_TYPE));
-    addColumns(document, analyzer, record.family, record.columns);
-    return document;
-  }
+//  public static Document convert(String rowId, Record record, Analyzer analyzer) {
+//    BlurUtil.validateRowIdAndRecord(rowId, record);
+//    Document document = new Document();
+//    document.add(new Field(BlurConstants.ROW_ID, rowId, ID_TYPE));
+//    document.add(new Field(BlurConstants.RECORD_ID, record.recordId, ID_TYPE));
+//    document.add(new Field(BlurConstants.FAMILY, record.family, ID_TYPE));
+//    addColumns(document, analyzer, record.family, record.columns);
+//    return document;
+//  }
 
   private Term createRowId(String id) {
     return new Term(BlurConstants.ROW_ID, id);
@@ -410,34 +409,34 @@ public class TransactionRecorder extends TimerTask implements Closeable {
     }
   }
 
-  public static boolean addColumns(Document document, BlurAnalyzer analyzer, String columnFamily, Iterable<Column> set) {
-    if (set == null) {
-      return false;
-    }
-    OUTER: for (Column column : set) {
-      String name = column.getName();
-      String value = column.value;
-      if (value == null || name == null) {
-        continue OUTER;
-      }
-      String fieldName = getFieldName(columnFamily, name);
-      FieldType fieldType = analyzer.getFieldType(fieldName);
-      Field field = analyzer.getField(fieldName, value, fieldType);
-      document.add(field);
-
-      if (analyzer.isFullTextField(fieldName)) {
-        document.add(new Field(SUPER, value, SUPER_FIELD_TYPE));
-      }
-      Set<String> subFieldNames = analyzer.getSubIndexNames(fieldName);
-      if (subFieldNames != null) {
-        for (String subFieldName : subFieldNames) {
-          FieldType subFieldType = analyzer.getFieldType(subFieldName);
-          document.add(analyzer.getField(subFieldName, value, subFieldType));
-        }
-      }
-    }
-    return true;
-  }
+//  public static boolean addColumns(Document document, Analyzer analyzer, String columnFamily, Iterable<Column> set) {
+//    if (set == null) {
+//      return false;
+//    }
+//    OUTER: for (Column column : set) {
+//      String name = column.getName();
+//      String value = column.value;
+//      if (value == null || name == null) {
+//        continue OUTER;
+//      }
+//      String fieldName = getFieldName(columnFamily, name);
+//      FieldType fieldType = analyzer.getFieldType(fieldName);
+//      Field field = analyzer.getField(fieldName, value, fieldType);
+//      document.add(field);
+//
+//      if (analyzer.isFullTextField(fieldName)) {
+//        document.add(new Field(SUPER, value, SUPER_FIELD_TYPE));
+//      }
+//      Set<String> subFieldNames = analyzer.getSubIndexNames(fieldName);
+//      if (subFieldNames != null) {
+//        for (String subFieldName : subFieldNames) {
+//          FieldType subFieldType = analyzer.getFieldType(subFieldName);
+//          document.add(analyzer.getField(subFieldName, value, subFieldType));
+//        }
+//      }
+//    }
+//    return true;
+//  }
 
   public static String getFieldName(String columnFamily, String name) {
     return columnFamily + SEP + name;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index 1427f1d..e9bcb0a 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -27,7 +27,7 @@ import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.thrift.generated.ScoreType;
@@ -50,7 +50,6 @@ public class TableContext {
 
   private Path tablePath;
   private Path walTablePath;
-  private BlurAnalyzer analyzer;
   private String defaultFieldName;
   private String table;
   private IndexDeletionPolicy indexDeletionPolicy;
@@ -61,6 +60,7 @@ public class TableContext {
   private long timeBetweenRefreshs;
   private ScoreType defaultScoreType;
   private Term defaultPrimeDocTerm;
+  private FieldManager fieldManager;
 
   private static ConcurrentHashMap<String, TableContext> cache = new ConcurrentHashMap<String, TableContext>();
 
@@ -90,7 +90,7 @@ public class TableContext {
     tableContext.configuration = configuration;
     tableContext.tablePath = new Path(tableDescriptor.getTableUri());
     tableContext.walTablePath = new Path(tableContext.tablePath, LOGS);
-    tableContext.analyzer = new BlurAnalyzer(tableDescriptor.getAnalyzerDefinition());
+    tableContext.fieldManager = null;
     tableContext.defaultFieldName = SUPER;
     tableContext.table = tableDescriptor.getName();
     tableContext.descriptor = tableDescriptor;
@@ -131,8 +131,8 @@ public class TableContext {
     return timeBetweenRefreshs;
   }
 
-  public BlurAnalyzer getAnalyzer() {
-    return analyzer;
+  public FieldManager getFieldManager() {
+    return fieldManager;
   }
 
   public String getTable() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-core/src/main/java/org/apache/blur/utils/HighlightHelper.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/HighlightHelper.java b/blur-core/src/main/java/org/apache/blur/utils/HighlightHelper.java
index 93915e6..fac107c 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/HighlightHelper.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/HighlightHelper.java
@@ -22,11 +22,12 @@ import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 
-import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.search.SuperQuery;
 import org.apache.blur.thrift.generated.Selector;
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field.Store;
@@ -71,7 +72,7 @@ public class HighlightHelper {
 
   public static List<Document> highlightDocuments(IndexReader reader, Term term,
       ResetableDocumentStoredFieldVisitor fieldSelector, Selector selector, Query highlightQuery,
-      BlurAnalyzer analyzer, String preTag, String postTag) throws IOException {
+      FieldManager fieldManager, String preTag, String postTag) throws IOException {
     IndexSearcher indexSearcher = new IndexSearcher(reader);
     int docFreq = reader.docFreq(term);
     BooleanQuery booleanQueryForFamily = null;
@@ -102,7 +103,7 @@ public class HighlightHelper {
       indexSearcher.doc(doc, fieldSelector);
       Document document = fieldSelector.getDocument();
       try {
-        document = highlight(doc, document, highlightQuery, analyzer, reader, preTag, postTag);
+        document = highlight(doc, document, highlightQuery, fieldManager, reader, preTag, postTag);
       } catch (InvalidTokenOffsetsException e) {
         LOG.error("Unknown error while tring to highlight", e);
       }
@@ -118,10 +119,12 @@ public class HighlightHelper {
    * @param preTag
    * @param postTag
    */
-  public static Document highlight(int docId, Document document, Query query, BlurAnalyzer analyzer,
+  public static Document highlight(int docId, Document document, Query query, FieldManager fieldManager,
       IndexReader reader, String preTag, String postTag) throws IOException, InvalidTokenOffsetsException {
     
     Query fixedQuery = fixSuperQuery(query, null);
+    
+    Analyzer analyzer = fieldManager.getAnalyzerForQuery();
 
     SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter(preTag, postTag);
     Document result = new Document();
@@ -135,7 +138,7 @@ public class HighlightHelper {
       Number numericValue = f.numericValue();
       
       Query fieldFixedQuery;
-      if (analyzer.isFullTextField(name)) {
+      if (fieldManager.isFieldLessIndexed(name)) {
         fieldFixedQuery = fixSuperQuery(query, name);  
       } else {
         fieldFixedQuery = fixedQuery;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
index f8190b2..5d343d1 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
@@ -24,7 +24,6 @@ import java.util.Map;
 import java.util.TreeMap;
 import java.util.UUID;
 
-import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.LuceneVersionConstant;
@@ -154,7 +153,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
       InterruptedException {
     int id = context.getTaskAttemptID().getTaskID().getId();
     TaskAttemptID taskAttemptID = context.getTaskAttemptID();
-    return new BlurRecordWriter(context.getConfiguration(), new BlurAnalyzer(), id, taskAttemptID.toString() + ".tmp");
+    return new BlurRecordWriter(context.getConfiguration(), id, taskAttemptID.toString() + ".tmp");
   }
 
   @Override
@@ -369,7 +368,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
 
     private Counter _rowDeleteCount;
 
-    public BlurRecordWriter(Configuration configuration, BlurAnalyzer blurAnalyzer, int attemptId, String tmpDirName)
+    public BlurRecordWriter(Configuration configuration, int attemptId, String tmpDirName)
         throws IOException {
 
       _indexLocally = BlurOutputFormat.isIndexLocally(configuration);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
index e14250a..7ddae37 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
@@ -29,6 +29,7 @@ import org.apache.blur.analysis.type.DoubleFieldTypeDefinition;
 import org.apache.blur.analysis.type.FloatFieldTypeDefinition;
 import org.apache.blur.analysis.type.IntFieldTypeDefinition;
 import org.apache.blur.analysis.type.LongFieldTypeDefinition;
+import org.apache.blur.analysis.type.NumericFieldTypeDefinition;
 import org.apache.blur.analysis.type.StoredFieldTypeDefinition;
 import org.apache.blur.analysis.type.StringFieldTypeDefinition;
 import org.apache.blur.analysis.type.TextFieldTypeDefinition;
@@ -38,9 +39,11 @@ import org.apache.blur.thrift.generated.Column;
 import org.apache.blur.thrift.generated.Record;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.AnalyzerWrapper;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.search.Query;
 
 public abstract class BaseFieldManager extends FieldManager {
 
@@ -50,8 +53,24 @@ public abstract class BaseFieldManager extends FieldManager {
   private Map<String, FieldTypeDefinition> _fieldNameToDefMap = new ConcurrentHashMap<String, FieldTypeDefinition>();
   private Map<String, Class<? extends FieldTypeDefinition>> _typeMap = new ConcurrentHashMap<String, Class<? extends FieldTypeDefinition>>();
   private Map<String, Set<String>> _columnToSubColumn = new ConcurrentHashMap<String, Set<String>>();
+  private Analyzer _baseAnalyzerForQuery;
 
-  public BaseFieldManager() {
+  public static FieldType ID_TYPE;
+  static {
+    ID_TYPE = new FieldType();
+    ID_TYPE.setIndexed(true);
+    ID_TYPE.setTokenized(false);
+    ID_TYPE.setOmitNorms(true);
+    ID_TYPE.setStored(true);
+    ID_TYPE.freeze();
+  }
+  private static final FieldType SUPER_FIELD_TYPE;
+  static {
+    SUPER_FIELD_TYPE = new FieldType(TextField.TYPE_NOT_STORED);
+    SUPER_FIELD_TYPE.setOmitNorms(true);
+  }
+
+  public BaseFieldManager(final Analyzer defaultAnalyzerForQuerying) {
     _typeMap.put(TextFieldTypeDefinition.NAME, TextFieldTypeDefinition.class);
     _typeMap.put(StringFieldTypeDefinition.NAME, StringFieldTypeDefinition.class);
     _typeMap.put(StoredFieldTypeDefinition.NAME, StoredFieldTypeDefinition.class);
@@ -59,15 +78,36 @@ public abstract class BaseFieldManager extends FieldManager {
     _typeMap.put(LongFieldTypeDefinition.NAME, LongFieldTypeDefinition.class);
     _typeMap.put(DoubleFieldTypeDefinition.NAME, DoubleFieldTypeDefinition.class);
     _typeMap.put(FloatFieldTypeDefinition.NAME, FloatFieldTypeDefinition.class);
+
+    _baseAnalyzerForQuery = new AnalyzerWrapper() {
+      @Override
+      protected Analyzer getWrappedAnalyzer(String fieldName) {
+        FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName);
+        if (fieldTypeDefinition == null) {
+          return defaultAnalyzerForQuerying;
+        }
+        return fieldTypeDefinition.getAnalyzerForQuery();
+      }
+
+      @Override
+      protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
+        return components;
+      }
+    };
   }
 
   @Override
-  public Iterable<? extends Field> getFields(String rowId, Record record) {
+  public List<Field> getFields(String rowId, Record record) {
     List<Field> fields = new ArrayList<Field>();
     String family = record.getFamily();
     List<Column> columns = record.getColumns();
     addDefaultFields(fields, rowId, record);
     for (Column column : columns) {
+      String name = column.getName();
+      String value = column.getValue();
+      if (value == null || name == null) {
+        continue;
+      }
       getAndAddFields(fields, family, column, getFieldTypeDefinition(family, column));
       Collection<String> subColumns = getSubColumns(family, column);
       if (subColumns != null) {
@@ -94,9 +134,9 @@ public abstract class BaseFieldManager extends FieldManager {
     validateNotNull(family, "family");
     validateNotNull(recordId, "recordId");
 
-    fields.add(new StringField(BlurConstants.FAMILY, family, Store.YES));
-    fields.add(new StringField(BlurConstants.ROW_ID, rowId, Store.YES));
-    fields.add(new StringField(BlurConstants.RECORD_ID, recordId, Store.YES));
+    fields.add(new Field(BlurConstants.FAMILY, family, ID_TYPE));
+    fields.add(new Field(BlurConstants.ROW_ID, rowId, ID_TYPE));
+    fields.add(new Field(BlurConstants.RECORD_ID, recordId, ID_TYPE));
   }
 
   private void validateNotNull(String value, String fieldName) {
@@ -116,7 +156,7 @@ public abstract class BaseFieldManager extends FieldManager {
   }
 
   private void addFieldLessIndex(List<Field> fields, String value) {
-    fields.add(new Field(BlurConstants.SUPER, value, TextFieldTypeDefinition.TYPE_NOT_STORED));
+    fields.add(new Field(BlurConstants.SUPER, value, SUPER_FIELD_TYPE));
   }
 
   private FieldTypeDefinition getFieldTypeDefinition(String family, Column column, String subName) {
@@ -220,4 +260,90 @@ public abstract class BaseFieldManager extends FieldManager {
     return fieldTypeDefinition.getAnalyzerForQuery();
   }
 
+  public void addColumnDefinitionInt(String family, String columnName) {
+    addColumnDefinition(family, columnName, null, false, IntFieldTypeDefinition.NAME, null);
+  }
+
+  public void addColumnDefinitionLong(String family, String columnName) {
+    addColumnDefinition(family, columnName, null, false, LongFieldTypeDefinition.NAME, null);
+  }
+
+  public void addColumnDefinitionFloat(String family, String columnName) {
+    addColumnDefinition(family, columnName, null, false, FloatFieldTypeDefinition.NAME, null);
+  }
+
+  public void addColumnDefinitionDouble(String family, String columnName) {
+    addColumnDefinition(family, columnName, null, false, DoubleFieldTypeDefinition.NAME, null);
+  }
+
+  public void addColumnDefinitionString(String family, String columnName) {
+    addColumnDefinition(family, columnName, null, false, StringFieldTypeDefinition.NAME, null);
+  }
+
+  public void addColumnDefinitionText(String family, String columnName) {
+    addColumnDefinition(family, columnName, null, false, TextFieldTypeDefinition.NAME, null);
+  }
+
+  public void addColumnDefinitionTextFieldLess(String family, String columnName) {
+    addColumnDefinition(family, columnName, null, true, TextFieldTypeDefinition.NAME, null);
+  }
+
+  @Override
+  public Analyzer getAnalyzerForQuery() {
+    return _baseAnalyzerForQuery;
+  }
+
+  @Override
+  public boolean checkSupportForFuzzyQuery(String field) {
+    FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
+    return fieldTypeDefinition.checkSupportForFuzzyQuery();
+  }
+
+  @Override
+  public boolean checkSupportForPrefixQuery(String field) {
+    FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
+    return fieldTypeDefinition.checkSupportForPrefixQuery();
+  }
+
+  @Override
+  public boolean checkSupportForWildcardQuery(String field) {
+    FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
+    return fieldTypeDefinition.checkSupportForWildcardQuery();
+  }
+
+  @Override
+  public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
+    FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
+    if (fieldTypeDefinition != null && fieldTypeDefinition.isNumeric()) {
+      NumericFieldTypeDefinition numericFieldTypeDefinition = (NumericFieldTypeDefinition) fieldTypeDefinition;
+      return numericFieldTypeDefinition.getNewRangeQuery(field, part1, part2, startInclusive, endInclusive);
+    }
+    return null;
+  }
+
+  @Override
+  public Query getTermQueryIfNumeric(String field, String text) {
+    FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
+    if (fieldTypeDefinition != null && fieldTypeDefinition.isNumeric()) {
+      NumericFieldTypeDefinition numericFieldTypeDefinition = (NumericFieldTypeDefinition) fieldTypeDefinition;
+      return numericFieldTypeDefinition.getNewRangeQuery(field, text, text, true, true);
+    }
+    return null;
+  }
+
+  @Override
+  public FieldTypeDefinition getFieldTypeDefinition(String field) {
+    return _fieldNameToDefMap.get(field);
+  }
+  
+  @Override
+  public boolean isFieldLessIndexed(String name) {
+    return false;
+  }
+
+  @Override
+  public Analyzer getAnalyzerForIndex() {
+    return null;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-query/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java b/blur-query/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
deleted file mode 100644
index 69535a1..0000000
--- a/blur-query/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
+++ /dev/null
@@ -1,531 +0,0 @@
-package org.apache.blur.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
-import static org.apache.blur.utils.BlurConstants.FAMILY;
-import static org.apache.blur.utils.BlurConstants.PRIME_DOC;
-import static org.apache.blur.utils.BlurConstants.RECORD_ID;
-import static org.apache.blur.utils.BlurConstants.ROW_ID;
-import static org.apache.blur.utils.BlurConstants.SUPER;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.lang.reflect.Constructor;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.blur.thirdparty.thrift_0_9_0.TException;
-import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TJSONProtocol;
-import org.apache.blur.thirdparty.thrift_0_9_0.transport.TMemoryBuffer;
-import org.apache.blur.thirdparty.thrift_0_9_0.transport.TMemoryInputTransport;
-import org.apache.blur.thrift.generated.AlternateColumnDefinition;
-import org.apache.blur.thrift.generated.AnalyzerDefinition;
-import org.apache.blur.thrift.generated.ColumnDefinition;
-import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.AnalyzerWrapper;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
-import org.apache.lucene.document.DoubleField;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.FloatField;
-import org.apache.lucene.document.IntField;
-import org.apache.lucene.document.LongField;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.util.Version;
-
-public final class BlurAnalyzer extends AnalyzerWrapper {
-
-  public enum TYPE {
-    LONG, DOUBLE, FLOAT, INTEGER, TEXT, STRING, STORED
-  }
-
-  @SuppressWarnings("serial")
-  private static Set<String> typeNameCache = new HashSet<String>() {
-    {
-      TYPE[] values = TYPE.values();
-      for (TYPE t : values) {
-        add(t.name());
-      }
-    }
-  };
-
-  private static final String STANDARD = "org.apache.blur.analysis.NoStopWordStandardAnalyzer";
-  public static final BlurAnalyzer BLANK_ANALYZER = new BlurAnalyzer(new KeywordAnalyzer());
-  private static Map<String, Class<? extends Analyzer>> aliases = new HashMap<String, Class<? extends Analyzer>>();
-
-  private Set<String> _subIndexNames = new HashSet<String>();
-  private Map<String, Set<String>> _subIndexNameLookups = new HashMap<String, Set<String>>();
-  private Map<String, Boolean> _fullTextFields = new HashMap<String, Boolean>();
-  private Map<String, Boolean> _fullTextColumnFamilies = new HashMap<String, Boolean>();
-  private AnalyzerDefinition _analyzerDefinition;
-  private Analyzer _defaultAnalyzer;
-  private Analyzer _keywordAnalyzer = new KeywordAnalyzer();
-  private Map<String, Analyzer> _analyzers = new HashMap<String, Analyzer>();
-  private Map<String, TYPE> _typeLookup = new HashMap<String, BlurAnalyzer.TYPE>();
-  private Map<String, FieldType> _fieldTypes = new HashMap<String, FieldType>();
-
-  public Set<String> getSubIndexNames(String indexName) {
-    return _subIndexNameLookups.get(indexName);
-  }
-
-  public BlurAnalyzer(Analyzer analyzer) {
-    _analyzerDefinition = new AnalyzerDefinition();
-    _defaultAnalyzer = analyzer;
-  }
-
-  public BlurAnalyzer(AnalyzerDefinition analyzerDefinition) {
-    _analyzerDefinition = analyzerDefinition;
-    ColumnDefinition defaultDefinition = analyzerDefinition.getDefaultDefinition();
-    if (defaultDefinition == null) {
-      defaultDefinition = new ColumnDefinition(STANDARD, true, null);
-      analyzerDefinition.setDefaultDefinition(defaultDefinition);
-    }
-    _defaultAnalyzer = getAnalyzerByClassName(defaultDefinition.getAnalyzerClassName(), aliases, null, null,
-        _fieldTypes);
-    _analyzers = new HashMap<String, Analyzer>();
-    _analyzers.put(ROW_ID, new KeywordAnalyzer());
-    _analyzers.put(RECORD_ID, new KeywordAnalyzer());
-    _analyzers.put(PRIME_DOC, new KeywordAnalyzer());
-    _analyzers.put(FAMILY, new KeywordAnalyzer());
-    _analyzers.put(SUPER, new WhitespaceAnalyzer(LUCENE_VERSION));
-    load(_analyzers, _analyzerDefinition.columnFamilyDefinitions, _fullTextFields, _subIndexNameLookups,
-        _subIndexNames, _fullTextColumnFamilies, _typeLookup, _fieldTypes);
-  }
-
-  public BlurAnalyzer() {
-    this(new AnalyzerDefinition());
-  }
-
-  private Analyzer getAnalyzer(String name) {
-    TYPE type = _typeLookup.get(name);
-    if (type == TYPE.STRING) {
-      return _keywordAnalyzer;
-    } else if (type == TYPE.STORED) {
-      throw new RuntimeException("Stored fields should never call this method.");
-    }
-    return _analyzers.get(name);
-  }
-
-  public TYPE getTypeLookup(String field) {
-    TYPE type = _typeLookup.get(field);
-    if (type == null) {
-      return TYPE.TEXT;
-    }
-    return type;
-  }
-
-  public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
-    TYPE type = _typeLookup.get(field);
-    if (type == null) {
-      return null;
-    }
-    FieldType fieldType = _fieldTypes.get(field);
-    switch (type) {
-
-    case STRING:
-      return null;
-
-    case INTEGER:
-      int integerPrecisionStep = fieldType.numericPrecisionStep();
-      int integerMin = Integer.parseInt(part1);
-      int integerMax = Integer.parseInt(part2);
-      return NumericRangeQuery.newIntRange(field, integerPrecisionStep, integerMin, integerMax, startInclusive,
-          endInclusive);
-
-    case DOUBLE:
-      int doublePrecisionStep = fieldType.numericPrecisionStep();
-      double doubleMin = Double.parseDouble(part1);
-      double doubleMax = Double.parseDouble(part2);
-      return NumericRangeQuery.newDoubleRange(field, doublePrecisionStep, doubleMin, doubleMax, startInclusive,
-          endInclusive);
-
-    case FLOAT:
-      int floatPrecisionStep = fieldType.numericPrecisionStep();
-      float floatMin = Float.parseFloat(part1);
-      float floatMax = Float.parseFloat(part2);
-      return NumericRangeQuery.newFloatRange(field, floatPrecisionStep, floatMin, floatMax, startInclusive,
-          endInclusive);
-
-    case LONG:
-      int longPrecisionStep = fieldType.numericPrecisionStep();
-      long longMin = Long.parseLong(part1);
-      long longMax = Long.parseLong(part2);
-      return NumericRangeQuery.newLongRange(field, longPrecisionStep, longMin, longMax, startInclusive, endInclusive);
-
-    default:
-      return null;
-    }
-
-  }
-
-  public boolean isFullTextField(String fieldName) {
-    Boolean b = _fullTextFields.get(fieldName);
-    if (b != null) {
-      return b;
-    }
-    String cf = getColumnFamily(fieldName);
-    if (cf == null) {
-      return false;
-    }
-    b = _fullTextColumnFamilies.get(cf);
-    if (b != null) {
-      return b;
-    }
-    ColumnDefinition defaultDefinition = _analyzerDefinition.getDefaultDefinition();
-    if (defaultDefinition != null && defaultDefinition.fullTextIndex) {
-      return true;
-    }
-    return false;
-  }
-
-  /**
-   * This method decides on the field type for the given field by name. Sub
-   * fields will also be passed in the fieldName such as fam1.col.sub1.
-   * 
-   * @param fieldName
-   * @return the {@link FieldType}
-   */
-  public FieldType getFieldType(String field) {
-    FieldType fieldType = _fieldTypes.get(field);
-    if (fieldType == null) {
-      fieldType = new FieldType(TextField.TYPE_STORED);
-      fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
-      fieldType.setOmitNorms(true);
-    }
-    if (isSubFieldName(field)) {
-      fieldType.setStored(false);
-    }
-    return fieldType;
-  }
-
-  /**
-   * Checks if the fieldName is a sub field or not.
-   * 
-   * @param fieldName
-   *          the field name to check.
-   * @return boolean
-   */
-  public boolean isSubFieldName(String fieldName) {
-    return _subIndexNames.contains(fieldName);
-  }
-
-  /**
-   * Get field will return the proper field for the given {@link FieldType}.
-   * 
-   * @param fieldName
-   *          the field name.
-   * @param value
-   *          the value.
-   * @param fieldType
-   *          the {@link FieldType}.
-   * @return the new {@link Field}.
-   */
-  public Field getField(String fieldName, String value, FieldType fieldType) {
-    TYPE type = _typeLookup.get(fieldName);
-    if (type == null) {
-      return new Field(fieldName, value, fieldType);
-    }
-    switch (type) {
-    case STORED:
-      return new StoredField(fieldName, value);
-    case STRING:
-      return new Field(fieldName, value, fieldType);
-    case INTEGER:
-      return new IntField(fieldName, Integer.parseInt(value), fieldType);
-    case DOUBLE:
-      return new DoubleField(fieldName, Double.parseDouble(value), fieldType);
-    case FLOAT:
-      return new FloatField(fieldName, Float.parseFloat(value), fieldType);
-    case LONG:
-      return new LongField(fieldName, Long.parseLong(value), fieldType);
-    default:
-      return new Field(fieldName, value, fieldType);
-    }
-  }
-
-  public String toJSON() {
-    TMemoryBuffer trans = new TMemoryBuffer(1024);
-    TJSONProtocol protocol = new TJSONProtocol(trans);
-    try {
-      _analyzerDefinition.write(protocol);
-    } catch (TException e) {
-      throw new RuntimeException(e);
-    }
-    trans.close();
-    byte[] array = trans.getArray();
-    return new String(array, 0, trans.length());
-  }
-
-  private String getColumnFamily(String fieldName) {
-    int index = fieldName.indexOf('.');
-    if (index < 0) {
-      return null;
-    }
-    return fieldName.substring(0, index);
-  }
-
-  public AnalyzerDefinition getAnalyzerDefinition() {
-    return _analyzerDefinition;
-  }
-
-  public void close() {
-
-  }
-
-  @Override
-  protected Analyzer getWrappedAnalyzer(String fieldName) {
-    Analyzer analyzer = getAnalyzer(fieldName);
-    return (analyzer != null) ? analyzer : _defaultAnalyzer;
-  }
-
-  @Override
-  protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
-    return components;
-  }
-
-  public static BlurAnalyzer create(File file) throws IOException {
-    FileInputStream inputStream = new FileInputStream(file);
-    try {
-      return create(inputStream);
-    } finally {
-      inputStream.close();
-    }
-  }
-
-  public static BlurAnalyzer create(InputStream inputStream) throws IOException {
-    TMemoryInputTransport trans = new TMemoryInputTransport(getBytes(inputStream));
-    TJSONProtocol protocol = new TJSONProtocol(trans);
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    try {
-      analyzerDefinition.read(protocol);
-    } catch (TException e) {
-      throw new RuntimeException(e);
-    }
-    trans.close();
-    return new BlurAnalyzer(analyzerDefinition);
-  }
-
-  public static BlurAnalyzer create(String jsonStr) throws IOException {
-    InputStream inputStream = new ByteArrayInputStream(jsonStr.getBytes());
-    try {
-      return create(inputStream);
-    } finally {
-      inputStream.close();
-    }
-  }
-
-  public static BlurAnalyzer create(Path path) throws IOException {
-    FileSystem fileSystem = FileSystem.get(path.toUri(), new Configuration());
-    FSDataInputStream inputStream = fileSystem.open(path);
-    try {
-      return create(inputStream);
-    } finally {
-      inputStream.close();
-    }
-  }
-
-  private static byte[] getBytes(InputStream inputStream) throws IOException {
-    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-    byte[] buffer = new byte[1024];
-    int num;
-    while ((num = inputStream.read(buffer)) != -1) {
-      outputStream.write(buffer, 0, num);
-    }
-    inputStream.close();
-    outputStream.close();
-    return outputStream.toByteArray();
-  }
-
-  private static void load(Map<String, Analyzer> analyzers, Map<String, ColumnFamilyDefinition> familyDefinitions,
-      Map<String, Boolean> fullTextFields, Map<String, Set<String>> subIndexNameLookups, Set<String> subIndexNames,
-      Map<String, Boolean> fullTextColumnFamilies, Map<String, TYPE> typeLookup, Map<String, FieldType> fieldTypes) {
-    if (familyDefinitions != null) {
-      for (String family : familyDefinitions.keySet()) {
-        ColumnFamilyDefinition familyDefinition = familyDefinitions.get(family);
-        ColumnDefinition defaultDefinition = familyDefinition.getDefaultDefinition();
-        if (defaultDefinition != null) {
-          fullTextColumnFamilies.put(family, defaultDefinition.isFullTextIndex());
-        }
-        load(family, familyDefinition, analyzers, fullTextFields, subIndexNameLookups, subIndexNames, typeLookup,
-            fieldTypes);
-      }
-    }
-  }
-
-  private static void load(String family, ColumnFamilyDefinition familyDefinition, Map<String, Analyzer> analyzers,
-      Map<String, Boolean> fullTextFields, Map<String, Set<String>> subIndexNameLookups, Set<String> subIndexNames,
-      Map<String, TYPE> typeLookup, Map<String, FieldType> fieldTypes) {
-    Map<String, ColumnDefinition> columnDefinitions = familyDefinition.getColumnDefinitions();
-    if (columnDefinitions != null) {
-      for (String column : columnDefinitions.keySet()) {
-        ColumnDefinition columnDefinition = columnDefinitions.get(column);
-        load(family, familyDefinition, column, columnDefinition, analyzers, fullTextFields, subIndexNameLookups,
-            subIndexNames, typeLookup, fieldTypes);
-      }
-    }
-  }
-
-  private static void load(String family, ColumnFamilyDefinition familyDefinition, String column,
-      ColumnDefinition columnDefinition, Map<String, Analyzer> analyzers, Map<String, Boolean> fullTextFields,
-      Map<String, Set<String>> subIndexNameLookups, Set<String> subIndexNames, Map<String, TYPE> typeLookup,
-      Map<String, FieldType> fieldTypes) {
-    Map<String, AlternateColumnDefinition> alternateColumnDefinitions = columnDefinition
-        .getAlternateColumnDefinitions();
-    if (alternateColumnDefinitions != null) {
-      for (String subColumn : alternateColumnDefinitions.keySet()) {
-        AlternateColumnDefinition alternateColumnDefinition = alternateColumnDefinitions.get(subColumn);
-        load(family, familyDefinition, column, columnDefinition, subColumn, alternateColumnDefinition, analyzers,
-            subIndexNameLookups, subIndexNames, typeLookup, fieldTypes);
-      }
-    }
-    String fieldName = family + "." + column;
-    Analyzer analyzer = getAnalyzerByClassName(columnDefinition.getAnalyzerClassName(), aliases, fieldName, typeLookup,
-        fieldTypes);
-    analyzers.put(fieldName, analyzer);
-    if (columnDefinition.isFullTextIndex()) {
-      fullTextFields.put(fieldName, Boolean.TRUE);
-    } else {
-      fullTextFields.put(fieldName, Boolean.FALSE);
-    }
-  }
-
-  private static void load(String family, ColumnFamilyDefinition familyDefinition, String column,
-      ColumnDefinition columnDefinition, String subColumn, AlternateColumnDefinition alternateColumnDefinition,
-      Map<String, Analyzer> analyzers, Map<String, Set<String>> subIndexNameLookups, Set<String> subIndexNames,
-      Map<String, TYPE> typeLookup, Map<String, FieldType> fieldTypes) {
-    String fieldName = family + "." + column + "." + subColumn;
-    Analyzer analyzer = getAnalyzerByClassName(alternateColumnDefinition.getAnalyzerClassName(), aliases, fieldName,
-        typeLookup, fieldTypes);
-    analyzers.put(fieldName, analyzer);
-    addSubField(fieldName, subIndexNameLookups);
-    subIndexNames.add(fieldName);
-  }
-
-  @SuppressWarnings("unchecked")
-  private static Analyzer getAnalyzerByClassName(String className, Map<String, Class<? extends Analyzer>> aliases,
-      String fieldName, Map<String, TYPE> typeLookup, Map<String, FieldType> fieldTypes) {
-    TYPE type = getType(className, fieldName, fieldTypes);
-    if (fieldName != null) {
-      typeLookup.put(fieldName, type);
-    }
-    if (type != TYPE.TEXT) {
-      return null;
-    }
-    try {
-      Class<? extends Analyzer> clazz = aliases.get(className);
-      if (clazz == null) {
-        clazz = (Class<? extends Analyzer>) Class.forName(className);
-      }
-      try {
-        return (Analyzer) clazz.newInstance();
-      } catch (Exception e) {
-        Constructor<?> constructor = clazz.getConstructor(new Class[] { Version.class });
-        return (Analyzer) constructor.newInstance(LUCENE_VERSION);
-      }
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  private static TYPE getType(String typeStr, String fieldName, Map<String, FieldType> fieldTypes) {
-    if (typeStr == null) {
-      return null;
-    }
-    String[] types = typeStr.split(",");
-    String type = types[0];
-    String typeUpper = type.toUpperCase();
-
-    TYPE t = null;
-    if (!typeNameCache.contains(typeUpper)) {
-      t = TYPE.TEXT;
-    } else {
-      t = TYPE.valueOf(typeUpper);
-    }
-
-    FieldType fieldType;
-    switch (t) {
-    case STORED:
-      fieldType = StoredField.TYPE;
-      break;
-    case STRING:
-      fieldType = new FieldType(StringField.TYPE_STORED);
-      break;
-    case LONG:
-      fieldType = new FieldType(LongField.TYPE_STORED);
-      if (types.length > 1) {
-        fieldType.setNumericPrecisionStep(Integer.parseInt(types[1]));
-      }
-      break;
-    case INTEGER:
-      fieldType = new FieldType(IntField.TYPE_STORED);
-      if (types.length > 1) {
-        fieldType.setNumericPrecisionStep(Integer.parseInt(types[1]));
-      }
-      break;
-    case FLOAT:
-      fieldType = new FieldType(FloatField.TYPE_STORED);
-      if (types.length > 1) {
-        fieldType.setNumericPrecisionStep(Integer.parseInt(types[1]));
-      }
-      break;
-    case DOUBLE:
-      fieldType = new FieldType(DoubleField.TYPE_STORED);
-      if (types.length > 1) {
-        fieldType.setNumericPrecisionStep(Integer.parseInt(types[1]));
-      }
-      break;
-    default:
-      fieldType = new FieldType(TextField.TYPE_STORED);
-      fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
-      fieldType.setOmitNorms(true);
-      break;
-    }
-    fieldTypes.put(fieldName, fieldType);
-    return t;
-  }
-
-  private static void addSubField(String name, Map<String, Set<String>> subIndexNameLookups) {
-    int lastIndexOf = name.lastIndexOf('.');
-    String mainFieldName = name.substring(0, lastIndexOf);
-    Set<String> set = subIndexNameLookups.get(mainFieldName);
-    if (set == null) {
-      set = new TreeSet<String>();
-      subIndexNameLookups.put(mainFieldName, set);
-    }
-    set.add(name);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
index 1269047..9629da9 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
@@ -16,11 +16,13 @@ package org.apache.blur.analysis;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.util.List;
 import java.util.Map;
 
 import org.apache.blur.thrift.generated.Record;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.search.Query;
 
 public abstract class FieldManager {
 
@@ -31,7 +33,7 @@ public abstract class FieldManager {
    * @param record
    * @return
    */
-  public abstract Iterable<? extends Field> getFields(String rowId, Record record);
+  public abstract List<Field> getFields(String rowId, Record record);
 
   /**
    * Adds a column definition.
@@ -84,4 +86,82 @@ public abstract class FieldManager {
    */
   public abstract boolean isValidColumnDefinition(String family, String columnName);
 
+  /**
+   * Gets an {@link Analyzer} for quering.
+   * 
+   * @return the analyzer
+   */
+  public abstract Analyzer getAnalyzerForQuery();
+
+  /**
+   * Checks if this field supports fuzzy queries.
+   * 
+   * @param field
+   *          the field name.
+   * @return boolean
+   */
+  public abstract boolean checkSupportForFuzzyQuery(String field);
+
+  /**
+   * Checks if this field supports prefix queries.
+   * 
+   * @param field
+   *          the field name.
+   * @return boolean
+   */
+  public abstract boolean checkSupportForPrefixQuery(String field);
+
+  /**
+   * Checks if this field supports wildcard queries.
+   * 
+   * @param field
+   *          the field name.
+   * @return boolean
+   */
+  public abstract boolean checkSupportForWildcardQuery(String field);
+
+  /**
+   * Gets a range query, if the method returns null the default term range query
+   * is used in the parser. This is used to create numeric range queries.
+   * 
+   * @param field
+   *          the field name.
+   * @param part1
+   *          the first part.
+   * @param part2
+   *          the second part.
+   * @param startInclusive
+   *          if the start is inclusive.
+   * @param endInclusive
+   *          if the end is inclusive.
+   * @return the new range query or null.
+   */
+  public abstract Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive,
+      boolean endInclusive);
+
+  /**
+   * This method should return a query (probably a range query) for numeric
+   * fields and null for other fields.
+   * 
+   * @param field
+   *          the field name.
+   * @param text
+   *          the text for the term.
+   * @return the query or null.
+   */
+  public abstract Query getTermQueryIfNumeric(String field, String text);
+
+  /**
+   * Gets the {@link FieldTypeDefinition} for the given field.
+   * 
+   * @param field
+   *          the field name.
+   * @return the {@link FieldTypeDefinition} or null if missing.
+   */
+  public abstract FieldTypeDefinition getFieldTypeDefinition(String field);
+
+  public abstract boolean isFieldLessIndexed(String name);
+
+  public abstract Analyzer getAnalyzerForIndex();
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
index e5877f8..1f24386 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
@@ -26,8 +26,6 @@ import org.apache.lucene.document.FieldType;
 
 public abstract class FieldTypeDefinition {
   
-  protected static final String NUMERIC_PRECISION_STEP = "numericPrecisionStep";
-
   protected boolean _fieldLessIndexing;
 
   /**
@@ -140,4 +138,13 @@ public abstract class FieldTypeDefinition {
   public void setFieldLessIndexing(boolean fieldLessIndexing) {
     _fieldLessIndexing = fieldLessIndexing;
   }
+
+  public abstract boolean checkSupportForFuzzyQuery();
+
+  public abstract boolean checkSupportForWildcardQuery();
+
+  public abstract boolean checkSupportForPrefixQuery();
+
+  public abstract boolean isNumeric();
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-query/src/main/java/org/apache/blur/analysis/type/DoubleFieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/DoubleFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/DoubleFieldTypeDefinition.java
index 41e086e..7bedbc8 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/type/DoubleFieldTypeDefinition.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/type/DoubleFieldTypeDefinition.java
@@ -18,15 +18,14 @@ package org.apache.blur.analysis.type;
  */
 import java.util.Map;
 
-import org.apache.blur.analysis.FieldTypeDefinition;
 import org.apache.blur.thrift.generated.Column;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.DoubleField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
 
-public class DoubleFieldTypeDefinition extends FieldTypeDefinition {
+public class DoubleFieldTypeDefinition extends NumericFieldTypeDefinition {
 
   public static final String NAME = "double";
   private FieldType _typeStored;
@@ -41,12 +40,12 @@ public class DoubleFieldTypeDefinition extends FieldTypeDefinition {
   public void configure(Map<String, String> properties) {
     String precisionStepStr = properties.get(NUMERIC_PRECISION_STEP);
     if (precisionStepStr != null) {
-      int precisionStep = Integer.parseInt(precisionStepStr);
+      _precisionStep = Integer.parseInt(precisionStepStr);
       _typeStored = new FieldType(DoubleField.TYPE_STORED);
-      _typeStored.setNumericPrecisionStep(precisionStep);
+      _typeStored.setNumericPrecisionStep(_precisionStep);
       _typeStored.freeze();
       _typeNotStored = new FieldType(DoubleField.TYPE_NOT_STORED);
-      _typeNotStored.setNumericPrecisionStep(precisionStep);
+      _typeNotStored.setNumericPrecisionStep(_precisionStep);
       _typeNotStored.freeze();
     } else {
       _typeStored = DoubleField.TYPE_STORED;
@@ -78,14 +77,10 @@ public class DoubleFieldTypeDefinition extends FieldTypeDefinition {
   }
 
   @Override
-  public Analyzer getAnalyzerForIndex() {
-    // shouldn't be used ever
-    return new KeywordAnalyzer();
-  }
-
-  @Override
-  public Analyzer getAnalyzerForQuery() {
-    return new KeywordAnalyzer();
+  public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
+    double p1 = Double.parseDouble(part1);
+    double p2 = Double.parseDouble(part2);
+    return NumericRangeQuery.newDoubleRange(field, _precisionStep, p1, p2, startInclusive, endInclusive);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-query/src/main/java/org/apache/blur/analysis/type/FloatFieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/FloatFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/FloatFieldTypeDefinition.java
index 6ba79fc..452518b 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/type/FloatFieldTypeDefinition.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/type/FloatFieldTypeDefinition.java
@@ -18,15 +18,14 @@ package org.apache.blur.analysis.type;
  */
 import java.util.Map;
 
-import org.apache.blur.analysis.FieldTypeDefinition;
 import org.apache.blur.thrift.generated.Column;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.FloatField;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
 
-public class FloatFieldTypeDefinition extends FieldTypeDefinition {
+public class FloatFieldTypeDefinition extends NumericFieldTypeDefinition {
 
   public static final String NAME = "float";
   private FieldType _typeStored;
@@ -41,12 +40,12 @@ public class FloatFieldTypeDefinition extends FieldTypeDefinition {
   public void configure(Map<String, String> properties) {
     String precisionStepStr = properties.get(NUMERIC_PRECISION_STEP);
     if (precisionStepStr != null) {
-      int precisionStep = Integer.parseInt(precisionStepStr);
+      _precisionStep = Integer.parseInt(precisionStepStr);
       _typeStored = new FieldType(FloatField.TYPE_STORED);
-      _typeStored.setNumericPrecisionStep(precisionStep);
+      _typeStored.setNumericPrecisionStep(_precisionStep);
       _typeStored.freeze();
       _typeNotStored = new FieldType(FloatField.TYPE_NOT_STORED);
-      _typeNotStored.setNumericPrecisionStep(precisionStep);
+      _typeNotStored.setNumericPrecisionStep(_precisionStep);
       _typeNotStored.freeze();
     } else {
       _typeStored = FloatField.TYPE_STORED;
@@ -78,14 +77,10 @@ public class FloatFieldTypeDefinition extends FieldTypeDefinition {
   }
 
   @Override
-  public Analyzer getAnalyzerForIndex() {
-    // shouldn't be used ever
-    return new KeywordAnalyzer();
-  }
-
-  @Override
-  public Analyzer getAnalyzerForQuery() {
-    return new KeywordAnalyzer();
+  public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
+    float p1 = Float.parseFloat(part1);
+    float p2 = Float.parseFloat(part2);
+    return NumericRangeQuery.newFloatRange(field, _precisionStep, p1, p2, startInclusive, endInclusive);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4ebe74ef/blur-query/src/main/java/org/apache/blur/analysis/type/IntFieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/IntFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/IntFieldTypeDefinition.java
index 61333e3..7cd16f0 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/type/IntFieldTypeDefinition.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/type/IntFieldTypeDefinition.java
@@ -18,15 +18,14 @@ package org.apache.blur.analysis.type;
  */
 import java.util.Map;
 
-import org.apache.blur.analysis.FieldTypeDefinition;
 import org.apache.blur.thrift.generated.Column;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.IntField;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
 
-public class IntFieldTypeDefinition extends FieldTypeDefinition {
+public class IntFieldTypeDefinition extends NumericFieldTypeDefinition {
 
   public static final String NAME = "int";
   private FieldType _typeStored;
@@ -41,12 +40,12 @@ public class IntFieldTypeDefinition extends FieldTypeDefinition {
   public void configure(Map<String, String> properties) {
     String precisionStepStr = properties.get(NUMERIC_PRECISION_STEP);
     if (precisionStepStr != null) {
-      int precisionStep = Integer.parseInt(precisionStepStr);
+      _precisionStep = Integer.parseInt(precisionStepStr);
       _typeStored = new FieldType(IntField.TYPE_STORED);
-      _typeStored.setNumericPrecisionStep(precisionStep);
+      _typeStored.setNumericPrecisionStep(_precisionStep);
       _typeStored.freeze();
       _typeNotStored = new FieldType(IntField.TYPE_NOT_STORED);
-      _typeNotStored.setNumericPrecisionStep(precisionStep);
+      _typeNotStored.setNumericPrecisionStep(_precisionStep);
       _typeNotStored.freeze();
     } else {
       _typeStored = IntField.TYPE_STORED;
@@ -78,14 +77,9 @@ public class IntFieldTypeDefinition extends FieldTypeDefinition {
   }
 
   @Override
-  public Analyzer getAnalyzerForIndex() {
-    // shouldn't be used ever
-    return new KeywordAnalyzer();
+  public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
+    int p1 = Integer.parseInt(part1);
+    int p2 = Integer.parseInt(part2);
+    return NumericRangeQuery.newIntRange(field, _precisionStep, p1, p2, startInclusive, endInclusive);
   }
-
-  @Override
-  public Analyzer getAnalyzerForQuery() {
-    return new KeywordAnalyzer();
-  }
-
 }


Mime
View raw message