incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [7/7] git commit: Created a HDFS field manager that stores the meta data about the table in ZK.
Date Tue, 16 Jul 2013 18:10:01 GMT
Created a HDFS field manager that stores the meta data about the table in ZK.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ae516a44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ae516a44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ae516a44

Branch: refs/heads/0.2.0-newtypesystem
Commit: ae516a442767b31d2c7e29b07a78aa08ec246dcf
Parents: ccf87ce
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Jul 16 14:09:10 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Jul 16 14:09:10 2013 -0400

----------------------------------------------------------------------
 .../manager/writer/TransactionRecorder.java     |   2 +-
 .../apache/blur/analysis/BaseFieldManager.java  | 113 +++++++----
 .../org/apache/blur/analysis/FieldManager.java  |  36 ++--
 .../apache/blur/analysis/HdfsFieldManager.java  |  65 ++++---
 .../apache/blur/analysis/LightBlurAnalyzer.java |  45 -----
 .../blur/lucene/search/BlurQueryParser.java     |  45 +++--
 .../blur/analysis/BaseFieldManagerTest.java     | 193 +++++++++++++++++++
 .../blur/analysis/BaseFiledManagerTest.java     | 187 ------------------
 .../blur/analysis/HdfsFieldManagerTest.java     | 110 +++++++++++
 .../blur/lucene/search/SuperParserTest.java     |  12 +-
 10 files changed, 491 insertions(+), 317 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
index ab9bfc6..994c687 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
@@ -356,7 +356,7 @@ public class TransactionRecorder extends TimerTask implements Closeable {
     }
   }
 
-  public static List<List<Field>> getDocs(Row row, FieldManager fieldManager) {
+  public static List<List<Field>> getDocs(Row row, FieldManager fieldManager) throws IOException {
     List<Record> records = row.records;
     int size = records.size();
     final String rowId = row.id;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
index b2f005d..0404f2c 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
@@ -16,6 +16,7 @@ package org.apache.blur.analysis;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -24,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
 import org.apache.blur.analysis.type.DoubleFieldTypeDefinition;
 import org.apache.blur.analysis.type.FloatFieldTypeDefinition;
@@ -50,8 +52,8 @@ public abstract class BaseFieldManager extends FieldManager {
   private static final Log LOG = LogFactory.getLog(BaseFieldManager.class);
   private static final Map<String, String> EMPTY_MAP = new HashMap<String, String>();;
 
-  private final Map<String, Set<String>> _columnToSubColumn = new ConcurrentHashMap<String, Set<String>>();
-  private final Map<String, FieldTypeDefinition> _fieldNameToDefMap = new ConcurrentHashMap<String, FieldTypeDefinition>();
+  private final ConcurrentMap<String, Set<String>> _columnToSubColumn = new ConcurrentHashMap<String, Set<String>>();
+  private final ConcurrentMap<String, FieldTypeDefinition> _fieldNameToDefMap = new ConcurrentHashMap<String, FieldTypeDefinition>();
 
   // This is loaded at object creation and never changed again.
   private final Map<String, Class<? extends FieldTypeDefinition>> _typeMap = new ConcurrentHashMap<String, Class<? extends FieldTypeDefinition>>();
@@ -87,7 +89,12 @@ public abstract class BaseFieldManager extends FieldManager {
     _baseAnalyzerForQuery = new AnalyzerWrapper() {
       @Override
       protected Analyzer getWrappedAnalyzer(String fieldName) {
-        FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName);
+        FieldTypeDefinition fieldTypeDefinition;
+        try {
+          fieldTypeDefinition = getFieldTypeDefinition(fieldName);
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
         if (fieldTypeDefinition == null) {
           return defaultAnalyzerForQuerying;
         }
@@ -103,7 +110,12 @@ public abstract class BaseFieldManager extends FieldManager {
     _baseAnalyzerForIndex = new AnalyzerWrapper() {
       @Override
       protected Analyzer getWrappedAnalyzer(String fieldName) {
-        FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName);
+        FieldTypeDefinition fieldTypeDefinition;
+        try {
+          fieldTypeDefinition = getFieldTypeDefinition(fieldName);
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
         if (fieldTypeDefinition == null) {
           throw new RuntimeException("Field [" + fieldName + "] not found.");
         }
@@ -118,7 +130,7 @@ public abstract class BaseFieldManager extends FieldManager {
   }
 
   @Override
-  public List<Field> getFields(String rowId, Record record) {
+  public List<Field> getFields(String rowId, Record record) throws IOException {
     List<Field> fields = new ArrayList<Field>();
     String family = record.getFamily();
     List<Column> columns = record.getColumns();
@@ -180,7 +192,7 @@ public abstract class BaseFieldManager extends FieldManager {
     fields.add(new Field(_fieldLessField, value, SUPER_FIELD_TYPE));
   }
 
-  private FieldTypeDefinition getFieldTypeDefinition(String family, Column column, String subName) {
+  private FieldTypeDefinition getFieldTypeDefinition(String family, Column column, String subName) throws IOException {
     return getFieldTypeDefinition(getSubColumnName(family, column, subName));
   }
 
@@ -200,13 +212,13 @@ public abstract class BaseFieldManager extends FieldManager {
     return _columnToSubColumn.get(getColumnName(family, column));
   }
 
-  private FieldTypeDefinition getFieldTypeDefinition(String family, Column column) {
+  private FieldTypeDefinition getFieldTypeDefinition(String family, Column column) throws IOException {
     return getFieldTypeDefinition(getColumnName(family, column));
   }
 
   @Override
   public boolean addColumnDefinition(String family, String columnName, String subColumnName, boolean fieldLessIndexing,
-      String fieldType, Map<String, String> props) {
+      String fieldType, Map<String, String> props) throws IOException {
     String baseFieldName = family + "." + columnName;
     String fieldName;
     if (subColumnName != null) {
@@ -230,10 +242,46 @@ public abstract class BaseFieldManager extends FieldManager {
     if (!tryToStore(fieldName, fieldLessIndexing, fieldType, props)) {
       return false;
     }
+    fieldTypeDefinition = newFieldTypeDefinition(fieldLessIndexing, fieldType, props);
+    registerFieldTypeDefinition(fieldName, fieldTypeDefinition);
+    return true;
+  }
+
+  protected void registerFieldTypeDefinition(String fieldName, FieldTypeDefinition fieldTypeDefinition) {
+    _fieldNameToDefMap.put(fieldName, fieldTypeDefinition);
+    String baseFieldName = getBaseFieldName(fieldName);
+    String subColumnName = getSubColumnName(fieldName);
+    if (subColumnName != null) {
+      Set<String> subColumnNames = _columnToSubColumn.get(baseFieldName);
+      if (subColumnNames == null) {
+        subColumnNames = getConcurrentSet();
+        _columnToSubColumn.put(baseFieldName, subColumnNames);
+      }
+      subColumnNames.add(subColumnName);
+    }
+  }
+
+  private String getSubColumnName(String fieldName) {
+    int lastIndexOf = fieldName.lastIndexOf('.');
+    int indexOf = fieldName.indexOf('.');
+    if (indexOf == lastIndexOf) {
+      return null;
+    }
+    return fieldName.substring(lastIndexOf + 1);
+  }
+
+  private String getBaseFieldName(String fieldName) {
+    int indexOf = fieldName.indexOf('.');
+    return fieldName.substring(0, indexOf);
+  }
+
+  protected FieldTypeDefinition newFieldTypeDefinition(boolean fieldLessIndexing, String fieldType,
+      Map<String, String> props) {
     Class<? extends FieldTypeDefinition> clazz = _typeMap.get(fieldType);
     if (clazz == null) {
       throw new IllegalArgumentException("FieldType of [" + fieldType + "] was not found.");
     }
+    FieldTypeDefinition fieldTypeDefinition;
     try {
       fieldTypeDefinition = clazz.newInstance();
     } catch (InstantiationException e) {
@@ -249,26 +297,20 @@ public abstract class BaseFieldManager extends FieldManager {
       fieldTypeDefinition.configure(props);
     }
     fieldTypeDefinition.setFieldLessIndexing(fieldLessIndexing);
-    _fieldNameToDefMap.put(fieldName, fieldTypeDefinition);
-    if (subColumnName != null) {
-      Set<String> subColumnNames = _columnToSubColumn.put(baseFieldName, getConcurrentSet());
-      subColumnNames.add(subColumnName);
-    }
-    return true;
+    return fieldTypeDefinition;
   }
 
   protected abstract boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType,
-      Map<String, String> props);
+      Map<String, String> props) throws IOException;
 
-  protected abstract void tryToLoad(String field, Map<String, FieldTypeDefinition> fieldNameToDefMap,
-      Map<String, Set<String>> columnToSubColumn);
+  protected abstract void tryToLoad(String fieldName) throws IOException;
 
   private Set<String> getConcurrentSet() {
     return Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
   }
 
   @Override
-  public boolean isValidColumnDefinition(String family, String columnName) {
+  public boolean isValidColumnDefinition(String family, String columnName) throws IOException {
     String fieldName = getColumnName(family, columnName);
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName);
     if (fieldTypeDefinition == null) {
@@ -278,7 +320,7 @@ public abstract class BaseFieldManager extends FieldManager {
   }
 
   @Override
-  public Analyzer getAnalyzerForIndex(String fieldName) {
+  public Analyzer getAnalyzerForIndex(String fieldName) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName);
     if (fieldTypeDefinition == null) {
       throw new AnalyzerNotFoundException(fieldName);
@@ -287,7 +329,7 @@ public abstract class BaseFieldManager extends FieldManager {
   }
 
   @Override
-  public Analyzer getAnalyzerForQuery(String fieldName) {
+  public Analyzer getAnalyzerForQuery(String fieldName) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName);
     if (fieldTypeDefinition == null) {
       throw new AnalyzerNotFoundException(fieldName);
@@ -295,31 +337,31 @@ public abstract class BaseFieldManager extends FieldManager {
     return fieldTypeDefinition.getAnalyzerForQuery();
   }
 
-  public void addColumnDefinitionInt(String family, String columnName) {
+  public void addColumnDefinitionInt(String family, String columnName) throws IOException {
     addColumnDefinition(family, columnName, null, false, IntFieldTypeDefinition.NAME, null);
   }
 
-  public void addColumnDefinitionLong(String family, String columnName) {
+  public void addColumnDefinitionLong(String family, String columnName) throws IOException {
     addColumnDefinition(family, columnName, null, false, LongFieldTypeDefinition.NAME, null);
   }
 
-  public void addColumnDefinitionFloat(String family, String columnName) {
+  public void addColumnDefinitionFloat(String family, String columnName) throws IOException {
     addColumnDefinition(family, columnName, null, false, FloatFieldTypeDefinition.NAME, null);
   }
 
-  public void addColumnDefinitionDouble(String family, String columnName) {
+  public void addColumnDefinitionDouble(String family, String columnName) throws IOException {
     addColumnDefinition(family, columnName, null, false, DoubleFieldTypeDefinition.NAME, null);
   }
 
-  public void addColumnDefinitionString(String family, String columnName) {
+  public void addColumnDefinitionString(String family, String columnName) throws IOException {
     addColumnDefinition(family, columnName, null, false, StringFieldTypeDefinition.NAME, null);
   }
 
-  public void addColumnDefinitionText(String family, String columnName) {
+  public void addColumnDefinitionText(String family, String columnName) throws IOException {
     addColumnDefinition(family, columnName, null, false, TextFieldTypeDefinition.NAME, null);
   }
 
-  public void addColumnDefinitionTextFieldLess(String family, String columnName) {
+  public void addColumnDefinitionTextFieldLess(String family, String columnName) throws IOException {
     addColumnDefinition(family, columnName, null, true, TextFieldTypeDefinition.NAME, null);
   }
 
@@ -334,31 +376,32 @@ public abstract class BaseFieldManager extends FieldManager {
   }
 
   @Override
-  public boolean isFieldLessIndexed(String field) {
+  public boolean isFieldLessIndexed(String field) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
     return fieldTypeDefinition.isFieldLessIndexed();
   }
 
   @Override
-  public boolean checkSupportForFuzzyQuery(String field) {
+  public boolean checkSupportForFuzzyQuery(String field) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
     return fieldTypeDefinition.checkSupportForFuzzyQuery();
   }
 
   @Override
-  public boolean checkSupportForPrefixQuery(String field) {
+  public boolean checkSupportForPrefixQuery(String field) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
     return fieldTypeDefinition.checkSupportForPrefixQuery();
   }
 
   @Override
-  public boolean checkSupportForWildcardQuery(String field) {
+  public boolean checkSupportForWildcardQuery(String field) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
     return fieldTypeDefinition.checkSupportForWildcardQuery();
   }
 
   @Override
-  public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
+  public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive)
+      throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
     if (fieldTypeDefinition != null && fieldTypeDefinition.isNumeric()) {
       NumericFieldTypeDefinition numericFieldTypeDefinition = (NumericFieldTypeDefinition) fieldTypeDefinition;
@@ -368,7 +411,7 @@ public abstract class BaseFieldManager extends FieldManager {
   }
 
   @Override
-  public Query getTermQueryIfNumeric(String field, String text) {
+  public Query getTermQueryIfNumeric(String field, String text) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
     if (fieldTypeDefinition != null && fieldTypeDefinition.isNumeric()) {
       NumericFieldTypeDefinition numericFieldTypeDefinition = (NumericFieldTypeDefinition) fieldTypeDefinition;
@@ -378,10 +421,10 @@ public abstract class BaseFieldManager extends FieldManager {
   }
 
   @Override
-  public FieldTypeDefinition getFieldTypeDefinition(String field) {
+  public FieldTypeDefinition getFieldTypeDefinition(String field) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = _fieldNameToDefMap.get(field);
     if (fieldTypeDefinition == null) {
-      tryToLoad(field, _fieldNameToDefMap, _columnToSubColumn);
+      tryToLoad(field);
       fieldTypeDefinition = _fieldNameToDefMap.get(field);
     }
     return fieldTypeDefinition;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
index 2cb7aea..2271726 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
@@ -16,6 +16,7 @@ package org.apache.blur.analysis;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -32,8 +33,9 @@ public abstract class FieldManager {
    * 
    * @param record
    * @return
+   * @throws IOException 
    */
-  public abstract List<Field> getFields(String rowId, Record record);
+  public abstract List<Field> getFields(String rowId, Record record) throws IOException;
 
   /**
    * Adds a column definition.
@@ -53,9 +55,10 @@ public abstract class FieldManager {
    * @param props
    *          the configuration properties for this column and type.
    * @return 
+   * @throws IOException 
    */
   public abstract boolean addColumnDefinition(String family, String columnName, String subColumnName,
-      boolean fieldLessIndexing, String fieldType, Map<String, String> props);
+      boolean fieldLessIndexing, String fieldType, Map<String, String> props) throws IOException;
 
   /**
    * Gets the analyzer for the indexing process.
@@ -63,8 +66,9 @@ public abstract class FieldManager {
    * @param fieldName
    *          the Lucene field name.
    * @return {@link Analyzer}.
+   * @throws IOException 
    */
-  public abstract Analyzer getAnalyzerForIndex(String fieldName);
+  public abstract Analyzer getAnalyzerForIndex(String fieldName) throws IOException;
 
   /**
    * Gets the analyzer for the querying.
@@ -72,8 +76,9 @@ public abstract class FieldManager {
    * @param fieldName
    *          the Lucene field name.
    * @return {@link Analyzer}.
+   * @throws IOException 
    */
-  public abstract Analyzer getAnalyzerForQuery(String fieldName);
+  public abstract Analyzer getAnalyzerForQuery(String fieldName) throws IOException;
 
   /**
    * Checks if there is a valid column definition for the given family and
@@ -84,8 +89,9 @@ public abstract class FieldManager {
    * @param columnName
    *          the column name.
    * @return boolean
+   * @throws IOException 
    */
-  public abstract boolean isValidColumnDefinition(String family, String columnName);
+  public abstract boolean isValidColumnDefinition(String family, String columnName) throws IOException;
 
   /**
    * Gets an {@link Analyzer} for quering.
@@ -100,8 +106,9 @@ public abstract class FieldManager {
    * @param field
    *          the field name.
    * @return boolean
+   * @throws IOException 
    */
-  public abstract boolean checkSupportForFuzzyQuery(String field);
+  public abstract boolean checkSupportForFuzzyQuery(String field) throws IOException;
 
   /**
    * Checks if this field supports prefix queries.
@@ -109,8 +116,9 @@ public abstract class FieldManager {
    * @param field
    *          the field name.
    * @return boolean
+   * @throws IOException 
    */
-  public abstract boolean checkSupportForPrefixQuery(String field);
+  public abstract boolean checkSupportForPrefixQuery(String field) throws IOException;
 
   /**
    * Checks if this field supports wildcard queries.
@@ -118,8 +126,9 @@ public abstract class FieldManager {
    * @param field
    *          the field name.
    * @return boolean
+   * @throws IOException 
    */
-  public abstract boolean checkSupportForWildcardQuery(String field);
+  public abstract boolean checkSupportForWildcardQuery(String field) throws IOException;
 
   /**
    * Gets a range query, if the method returns null the default term range query
@@ -136,9 +145,10 @@ public abstract class FieldManager {
    * @param endInclusive
    *          if the end is inclusive.
    * @return the new range query or null.
+   * @throws IOException 
    */
   public abstract Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive,
-      boolean endInclusive);
+      boolean endInclusive) throws IOException;
 
   /**
    * This method should return a query (probably a range query) for numeric
@@ -149,8 +159,9 @@ public abstract class FieldManager {
    * @param text
    *          the text for the term.
    * @return the query or null.
+   * @throws IOException 
    */
-  public abstract Query getTermQueryIfNumeric(String field, String text);
+  public abstract Query getTermQueryIfNumeric(String field, String text) throws IOException;
 
   /**
    * Gets the {@link FieldTypeDefinition} for the given field.
@@ -158,10 +169,11 @@ public abstract class FieldManager {
    * @param field
    *          the field name.
    * @return the {@link FieldTypeDefinition} or null if missing.
+   * @throws IOException 
    */
-  public abstract FieldTypeDefinition getFieldTypeDefinition(String field);
+  public abstract FieldTypeDefinition getFieldTypeDefinition(String field) throws IOException;
 
-  public abstract boolean isFieldLessIndexed(String name);
+  public abstract boolean isFieldLessIndexed(String name) throws IOException;
 
   public abstract Analyzer getAnalyzerForIndex();
   

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
index 3d5d5c2..9669e2e 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
@@ -17,14 +17,15 @@ package org.apache.blur.analysis;
  * limitations under the License.
  */
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
-import java.util.Set;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -32,6 +33,9 @@ import org.apache.lucene.analysis.Analyzer;
 
 public class HdfsFieldManager extends BaseFieldManager {
 
+  private static final String FIELD_TYPE = "fieldType";
+  private static final String FIELD_LESS_INDEXING = "fieldLessIndexing";
+
   private static final Log LOG = LogFactory.getLog(HdfsFieldManager.class);
 
   private final Configuration _configuration;
@@ -47,43 +51,58 @@ public class HdfsFieldManager extends BaseFieldManager {
   }
 
   @Override
-  protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props) {
+  protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props)
+      throws IOException {
+    LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]",
+        fieldName, fieldLessIndexing, fieldType, props);
     Properties properties = new Properties();
-    properties.put("fieldLessIndexing", fieldLessIndexing);
-    properties.put("fieldType", fieldType);
+    properties.setProperty(FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexing));
+    properties.setProperty(FIELD_TYPE, fieldType);
     if (props != null) {
       for (Entry<String, String> e : props.entrySet()) {
         properties.put(e.getKey(), e.getValue());
       }
     }
-    Path path = new Path(_storagePath, fieldName);
-    try {
-      if (_fileSystem.exists(path)) {
-        return false;
-      }
-    } catch (IOException e) {
-      LOG.error("Could not check filesystem for existence of path [{0}]", e, path);
-      return false;
-    }
-    try {
-      FSDataOutputStream outputStream = _fileSystem.create(path, false);
-      properties.store(outputStream, getComments());
-      outputStream.close();
-    } catch (IOException e) {
-      LOG.error("Could not create field [" + fieldName + "]", e);
+    Path path = getFieldPath(fieldName);
+    if (_fileSystem.exists(path)) {
+      LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props);
       return false;
     }
+    FSDataOutputStream outputStream = _fileSystem.create(path, false);
+    properties.store(outputStream, getComments());
+    outputStream.close();
     return true;
   }
 
+  private Path getFieldPath(String fieldName) {
+    return new Path(_storagePath, fieldName);
+  }
+
   private String getComments() {
-    return null;
+    return "This file is generated from Apache Blur to store meta data about field types.  DO NOT MODIFIY!";
   }
 
   @Override
-  protected void tryToLoad(String field, Map<String, FieldTypeDefinition> fieldNameToDefMap,
-      Map<String, Set<String>> columnToSubColumn) {
-    
+  protected void tryToLoad(String fieldName) throws IOException {
+    Path path = getFieldPath(fieldName);
+    if (!_fileSystem.exists(path)) {
+      return;
+    }
+    FSDataInputStream inputStream = _fileSystem.open(path);
+    Properties props = new Properties();
+    props.load(inputStream);
+    inputStream.close();
+    boolean fieldLessIndexing = Boolean.parseBoolean(props.getProperty(FIELD_LESS_INDEXING));
+    String fieldType = props.getProperty(FIELD_TYPE);
+    FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldLessIndexing, fieldType, toMap(props));
+    registerFieldTypeDefinition(fieldName, fieldTypeDefinition);
   }
 
+  private Map<String, String> toMap(Properties props) {
+    Map<String, String> result = new HashMap<String, String>();
+    for (Entry<Object, Object> e : props.entrySet()) {
+      result.put(e.getKey().toString(), e.getValue().toString());
+    }
+    return result;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java b/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java
deleted file mode 100644
index 1939119..0000000
--- a/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.apache.blur.analysis;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.AnalyzerWrapper;
-
-public class LightBlurAnalyzer extends AnalyzerWrapper {
-
-  private final FieldManager _fieldManager;
-  private final boolean _usedForIndexing;
-
-  public LightBlurAnalyzer(boolean usedForIndexing, FieldManager fieldManager) {
-    _usedForIndexing = usedForIndexing;
-    _fieldManager = fieldManager;
-  }
-
-  @Override
-  protected Analyzer getWrappedAnalyzer(String fieldName) {
-    if (_usedForIndexing) {
-      return _fieldManager.getAnalyzerForIndex(fieldName);
-    } else {
-      return _fieldManager.getAnalyzerForQuery(fieldName);
-    }
-  }
-
-  @Override
-  protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
-    return components;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java b/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java
index 26762a6..169ada4 100644
--- a/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java
+++ b/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java
@@ -16,6 +16,7 @@ package org.apache.blur.lucene.search;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
@@ -45,9 +46,13 @@ public class BlurQueryParser extends QueryParser {
   @Override
   protected Query newFuzzyQuery(Term term, float minimumSimilarity, int prefixLength) {
     String field = term.field();
-    if (!_fieldManager.checkSupportForFuzzyQuery(field)) {
-      throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field)
-          + "] which does not support fuzzy queries.");
+    try {
+      if (!_fieldManager.checkSupportForFuzzyQuery(field)) {
+        throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field)
+            + "] which does not support fuzzy queries.");
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
     }
     return addField(super.newFuzzyQuery(term, minimumSimilarity, prefixLength), term.field());
   }
@@ -85,16 +90,25 @@ public class BlurQueryParser extends QueryParser {
   @Override
   protected Query newPrefixQuery(Term prefix) {
     String field = prefix.field();
-    if (!_fieldManager.checkSupportForPrefixQuery(field)) {
-      throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field)
-          + "] which does not support prefix queries.");
+    try {
+      if (!_fieldManager.checkSupportForPrefixQuery(field)) {
+        throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field)
+            + "] which does not support prefix queries.");
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
     }
     return addField(super.newPrefixQuery(prefix), field);
   }
 
   @Override
   protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
-    Query q = _fieldManager.getNewRangeQuery(field, part1, part2, startInclusive, endInclusive);
+    Query q;
+    try {
+      q = _fieldManager.getNewRangeQuery(field, part1, part2, startInclusive, endInclusive);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
     if (q != null) {
       return addField(q, field);
     }
@@ -104,7 +118,12 @@ public class BlurQueryParser extends QueryParser {
   @Override
   protected Query newTermQuery(Term term) {
     String field = term.field();
-    Query q = _fieldManager.getTermQueryIfNumeric(field, term.text());
+    Query q;
+    try {
+      q = _fieldManager.getTermQueryIfNumeric(field, term.text());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
     if (q != null) {
       return addField(q, field);
     }
@@ -117,9 +136,13 @@ public class BlurQueryParser extends QueryParser {
       return new MatchAllDocsQuery();
     }
     String field = t.field();
-    if (!_fieldManager.checkSupportForWildcardQuery(field)) {
-      throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field)
-          + "] which does not support wildcard queries.");
+    try {
+      if (!_fieldManager.checkSupportForWildcardQuery(field)) {
+        throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field)
+            + "] which does not support wildcard queries.");
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
     }
     return addField(super.newWildcardQuery(t), t.field());
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java
new file mode 100644
index 0000000..cc44e84
--- /dev/null
+++ b/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java
@@ -0,0 +1,193 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.blur.analysis.type.TextFieldTypeDefinition;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.utils.BlurConstants;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.Field;
+import org.junit.Test;
+
+public class BaseFieldManagerTest {
+
+  public static final String _fieldLessField = BlurConstants.SUPER;
+
+  @Test
+  public void testFiledManager() throws IOException {
+    BaseFieldManager memoryFieldManager = newFieldManager(true);
+    memoryFieldManager.addColumnDefinition("fam1", "col1", null, true, "text", null);
+
+    Record record = new Record();
+    record.setFamily("fam1");
+    record.setRecordId("1213");
+    record.addToColumns(new Column("col1", "value1"));
+
+    List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"),
+        newTextFieldNoStore(_fieldLessField, "value1"));
+
+    int c = 0;
+    for (Field field : memoryFieldManager.getFields("1", record)) {
+      assertFieldEquals(fields.get(c++), field);
+    }
+  }
+
+  @Test
+  public void testFiledManagerMultipleColumnsSameName() throws IOException {
+    BaseFieldManager memoryFieldManager = newFieldManager(true);
+    memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null);
+
+    Record record = new Record();
+    record.setFamily("fam1");
+    record.setRecordId("1213");
+    record.addToColumns(new Column("col1", "value1"));
+    record.addToColumns(new Column("col1", "value2"));
+
+    List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"),
+        newTextField("fam1.col1", "value2"));
+
+    int c = 0;
+    for (Field field : memoryFieldManager.getFields("1", record)) {
+      assertFieldEquals(fields.get(c++), field);
+    }
+
+  }
+
+  @Test
+  public void testFiledManagerMultipleColumnsDifferentNames() throws IOException {
+    BaseFieldManager memoryFieldManager = newFieldManager(true);
+    memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null);
+    memoryFieldManager.addColumnDefinition("fam1", "col2", null, true, "text", null);
+
+    Record record = new Record();
+    record.setFamily("fam1");
+    record.setRecordId("1213");
+    record.addToColumns(new Column("col1", "value1"));
+    record.addToColumns(new Column("col2", "value2"));
+
+    List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"),
+        newTextField("fam1.col2", "value2"), newTextFieldNoStore(_fieldLessField, "value2"));
+
+    int c = 0;
+    for (Field field : memoryFieldManager.getFields("1", record)) {
+      assertFieldEquals(fields.get(c++), field);
+    }
+  }
+
+  @Test
+  public void testFiledManagerMultipleColumnsDifferentNamesDifferentFamilies() throws IOException {
+    BaseFieldManager memoryFieldManager = newFieldManager(true);
+    memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null);
+    memoryFieldManager.addColumnDefinition("fam2", "col2", null, false, "text", null);
+
+    Record record1 = new Record();
+    record1.setFamily("fam1");
+    record1.setRecordId("1213");
+    record1.addToColumns(new Column("col1", "value1"));
+
+    List<Field> fields1 = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"));
+    int c1 = 0;
+    for (Field field : memoryFieldManager.getFields("1", record1)) {
+      assertFieldEquals(fields1.get(c1++), field);
+    }
+
+    Record record2 = new Record();
+    record2.setFamily("fam2");
+    record2.setRecordId("1213");
+    record2.addToColumns(new Column("col2", "value1"));
+
+    List<Field> fields2 = getFields("fam2", "1", "1213", newTextField("fam2.col2", "value1"));
+    int c2 = 0;
+    for (Field field : memoryFieldManager.getFields("1", record2)) {
+      assertFieldEquals(fields2.get(c2++), field);
+    }
+  }
+
+  @Test
+  public void testFiledManagerSubNameWithMainColumnNameNoParent() throws IOException {
+    BaseFieldManager memoryFieldManager = newFieldManager(true);
+    try {
+      memoryFieldManager.addColumnDefinition("fam1", "col1", "sub1", false, "text", null);
+      fail("Should throw IllegalArgumentException");
+    } catch (IllegalArgumentException e) {
+    }
+  }
+
+  @Test
+  public void testFiledManagerSubNameWithMainColumnNameNoFieldLess() throws IOException {
+    BaseFieldManager memoryFieldManager = newFieldManager(true);
+    memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null);
+    try {
+      memoryFieldManager.addColumnDefinition("fam1", "col1", "sub1", true, "text", null);
+      fail("Should throw IllegalArgumentException");
+    } catch (IllegalArgumentException e) {
+    }
+  }
+
+  private List<Field> getFields(String family, String rowId, String recordId, Field... fields) {
+    List<Field> fieldLst = new ArrayList<Field>();
+    fieldLst.add(new Field("family", family, BaseFieldManager.ID_TYPE));
+    fieldLst.add(new Field("rowid", rowId, BaseFieldManager.ID_TYPE));
+    fieldLst.add(new Field("recordid", recordId, BaseFieldManager.ID_TYPE));
+    for (Field field : fields) {
+      fieldLst.add(field);
+    }
+    return fieldLst;
+  }
+
+  private Field newTextField(String name, String value) {
+    return new Field(name, value, TextFieldTypeDefinition.TYPE_STORED);
+  }
+
+  private Field newTextFieldNoStore(String name, String value) {
+    return new Field(name, value, TextFieldTypeDefinition.TYPE_NOT_STORED);
+  }
+
+  private void assertFieldEquals(Field expected, Field actual) {
+    assertEquals("Names did not match Expected [" + expected + "] Actual [" + actual + "]", expected.name(),
+        actual.name());
+    assertEquals("Values did not match Expected [" + expected + "] Actual [" + actual + "]", expected.stringValue(),
+        actual.stringValue());
+    assertEquals("FileTypes did not match Expected [" + expected + "] Actual [" + actual + "]", expected.fieldType()
+        .toString(), actual.fieldType().toString());
+  }
+
+  protected BaseFieldManager newFieldManager(boolean create) throws IOException {
+    return new BaseFieldManager(_fieldLessField, new KeywordAnalyzer()) {
+      @Override
+      protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType,
+          Map<String, String> props) {
+        return true;
+      }
+
+      @Override
+      protected void tryToLoad(String field) {
+
+      }
+    };
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/test/java/org/apache/blur/analysis/BaseFiledManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-query/src/test/java/org/apache/blur/analysis/BaseFiledManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/BaseFiledManagerTest.java
deleted file mode 100644
index 19a7586..0000000
--- a/blur-query/src/test/java/org/apache/blur/analysis/BaseFiledManagerTest.java
+++ /dev/null
@@ -1,187 +0,0 @@
-package org.apache.blur.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.blur.analysis.type.TextFieldTypeDefinition;
-import org.apache.blur.thrift.generated.Column;
-import org.apache.blur.thrift.generated.Record;
-import org.apache.blur.utils.BlurConstants;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.document.Field;
-import org.junit.Test;
-
-public class BaseFiledManagerTest {
-
-  private static final String _fieldLessField = BlurConstants.SUPER;
-
-  @Test
-  public void testFiledManager() {
-    BaseFieldManager memoryFieldManager = newBaseFieldManager();
-    memoryFieldManager.addColumnDefinition("fam1", "col1", null, true, "text", null);
-
-    Record record = new Record();
-    record.setFamily("fam1");
-    record.setRecordId("1213");
-    record.addToColumns(new Column("col1", "value1"));
-
-    List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"),
-        newTextFieldNoStore(_fieldLessField, "value1"));
-
-    int c = 0;
-    for (Field field : memoryFieldManager.getFields("1", record)) {
-      assertFieldEquals(fields.get(c++), field);
-    }
-  }
-
-  @Test
-  public void testFiledManagerMultipleColumnsSameName() {
-    BaseFieldManager memoryFieldManager = newBaseFieldManager();
-    memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null);
-
-    Record record = new Record();
-    record.setFamily("fam1");
-    record.setRecordId("1213");
-    record.addToColumns(new Column("col1", "value1"));
-    record.addToColumns(new Column("col1", "value2"));
-
-    List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"),
-        newTextField("fam1.col1", "value2"));
-
-    int c = 0;
-    for (Field field : memoryFieldManager.getFields("1", record)) {
-      assertFieldEquals(fields.get(c++), field);
-    }
-
-  }
-
-  @Test
-  public void testFiledManagerMultipleColumnsDifferentNames() {
-    BaseFieldManager memoryFieldManager = newBaseFieldManager();
-    memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null);
-    memoryFieldManager.addColumnDefinition("fam1", "col2", null, true, "text", null);
-
-    Record record = new Record();
-    record.setFamily("fam1");
-    record.setRecordId("1213");
-    record.addToColumns(new Column("col1", "value1"));
-    record.addToColumns(new Column("col2", "value2"));
-
-    List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"),
-        newTextField("fam1.col2", "value2"), newTextFieldNoStore(_fieldLessField, "value2"));
-
-    int c = 0;
-    for (Field field : memoryFieldManager.getFields("1", record)) {
-      assertFieldEquals(fields.get(c++), field);
-    }
-  }
-
-  @Test
-  public void testFiledManagerMultipleColumnsDifferentNamesDifferentFamilies() {
-    BaseFieldManager memoryFieldManager = newBaseFieldManager();
-    memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null);
-    memoryFieldManager.addColumnDefinition("fam2", "col2", null, false, "text", null);
-
-    Record record1 = new Record();
-    record1.setFamily("fam1");
-    record1.setRecordId("1213");
-    record1.addToColumns(new Column("col1", "value1"));
-
-    List<Field> fields1 = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"));
-    int c1 = 0;
-    for (Field field : memoryFieldManager.getFields("1", record1)) {
-      assertFieldEquals(fields1.get(c1++), field);
-    }
-
-    Record record2 = new Record();
-    record2.setFamily("fam2");
-    record2.setRecordId("1213");
-    record2.addToColumns(new Column("col2", "value1"));
-
-    List<Field> fields2 = getFields("fam2", "1", "1213", newTextField("fam2.col2", "value1"));
-    int c2 = 0;
-    for (Field field : memoryFieldManager.getFields("1", record2)) {
-      assertFieldEquals(fields2.get(c2++), field);
-    }
-  }
-
-  @Test
-  public void testFiledManagerSubNameWithMainColumnNameNoParent() {
-    BaseFieldManager memoryFieldManager = newBaseFieldManager();
-    try {
-      memoryFieldManager.addColumnDefinition("fam1", "col1", "sub1", false, "text", null);
-      fail("Should throw IllegalArgumentException");
-    } catch (IllegalArgumentException e) {
-    }
-  }
-
-  @Test
-  public void testFiledManagerSubNameWithMainColumnNameNoFieldLess() {
-    BaseFieldManager memoryFieldManager = newBaseFieldManager();
-    memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null);
-    try {
-      memoryFieldManager.addColumnDefinition("fam1", "col1", "sub1", true, "text", null);
-      fail("Should throw IllegalArgumentException");
-    } catch (IllegalArgumentException e) {
-    }
-  }
-
-  private List<Field> getFields(String family, String rowId, String recordId, Field... fields) {
-    List<Field> fieldLst = new ArrayList<Field>();
-    fieldLst.add(new Field("family", family, BaseFieldManager.ID_TYPE));
-    fieldLst.add(new Field("rowid", rowId, BaseFieldManager.ID_TYPE));
-    fieldLst.add(new Field("recordid", recordId, BaseFieldManager.ID_TYPE));
-    for (Field field : fields) {
-      fieldLst.add(field);
-    }
-    return fieldLst;
-  }
-
-  private Field newTextField(String name, String value) {
-    return new Field(name, value, TextFieldTypeDefinition.TYPE_STORED);
-  }
-
-  private Field newTextFieldNoStore(String name, String value) {
-    return new Field(name, value, TextFieldTypeDefinition.TYPE_NOT_STORED);
-  }
-
-  private void assertFieldEquals(Field expected, Field actual) {
-    assertEquals("Names did not match Expected [" + expected + "] Actual [" + actual + "]", expected.name(),
-        actual.name());
-    assertEquals("Values did not match Expected [" + expected + "] Actual [" + actual + "]", expected.stringValue(),
-        actual.stringValue());
-    assertEquals("FileTypes did not match Expected [" + expected + "] Actual [" + actual + "]", expected.fieldType()
-        .toString(), actual.fieldType().toString());
-  }
-
-  private BaseFieldManager newBaseFieldManager() {
-    return new BaseFieldManager(_fieldLessField, new KeywordAnalyzer()) {
-      @Override
-      protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType,
-          Map<String, String> props) {
-        return true;
-      }
-    };
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
new file mode 100644
index 0000000..ebf690a
--- /dev/null
+++ b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java
@@ -0,0 +1,110 @@
+package org.apache.blur.analysis;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.junit.Test;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+public class HdfsFieldManagerTest extends BaseFieldManagerTest {
+  @Override
+  protected BaseFieldManager newFieldManager(boolean create) throws IOException {
+    Configuration config = new Configuration();
+    Path path = new Path("./target/tmp/HdfsFieldManagerTest/meta");
+    FileSystem fileSystem = path.getFileSystem(config);
+    if (create) {
+      fileSystem.delete(path, true);
+    }
+    return new HdfsFieldManager(_fieldLessField, new KeywordAnalyzer(), path, config);
+  }
+
+  @SuppressWarnings("unchecked")
+  @Test
+  public void testStoreMetaData() throws IOException {
+    BaseFieldManager memoryFieldManager = newFieldManager(true);
+    memoryFieldManager.addColumnDefinition("fam1", "col1", null, true, "text", null);
+    memoryFieldManager.addColumnDefinition("fam2", "col2", null, false, "string", null);
+    memoryFieldManager.addColumnDefinition("fam2", "col2", "a", false, "text", null);
+    memoryFieldManager.addColumnDefinition("fam2", "col2", "b", false, "text", newMap(e("a", "b")));
+    memoryFieldManager.addColumnDefinition("fam2", "col3", null, false, "int", null);
+
+    BaseFieldManager memoryFieldManager2 = newFieldManager(false);
+    FieldTypeDefinition fieldTypeDefinition1 = memoryFieldManager2.getFieldTypeDefinition("fam1.col1");
+    assertNotNull(fieldTypeDefinition1);
+    assertTrue(fieldTypeDefinition1.isFieldLessIndexed());
+    assertFalse(fieldTypeDefinition1.isNumeric());
+
+    FieldTypeDefinition fieldTypeDefinition2 = memoryFieldManager2.getFieldTypeDefinition("fam2.col2");
+    assertNotNull(fieldTypeDefinition2);
+    assertFalse(fieldTypeDefinition2.isFieldLessIndexed());
+    assertFalse(fieldTypeDefinition2.isNumeric());
+    
+    FieldTypeDefinition fieldTypeDefinition3 = memoryFieldManager2.getFieldTypeDefinition("fam2.col2.a");
+    assertNotNull(fieldTypeDefinition3);
+    assertFalse(fieldTypeDefinition3.isFieldLessIndexed());
+    assertFalse(fieldTypeDefinition3.isNumeric());
+    
+    FieldTypeDefinition fieldTypeDefinition4 = memoryFieldManager2.getFieldTypeDefinition("fam2.col2.b");
+    assertNotNull(fieldTypeDefinition4);
+    assertFalse(fieldTypeDefinition4.isFieldLessIndexed());
+    assertFalse(fieldTypeDefinition4.isNumeric());
+
+    FieldTypeDefinition fieldTypeDefinition5 = memoryFieldManager2.getFieldTypeDefinition("fam2.col3");
+    assertNotNull(fieldTypeDefinition5);
+    assertFalse(fieldTypeDefinition5.isFieldLessIndexed());
+    assertTrue(fieldTypeDefinition5.isNumeric());
+
+    assertNull(memoryFieldManager2.getFieldTypeDefinition("fam2.col2.c"));
+  }
+
+  private static <K, V> Map<K, V> newMap(Entry<K, V>... es) {
+    Map<K, V> map = new HashMap<K, V>();
+    for (Entry<K, V> e : es) {
+      map.put(e.getKey(), e.getValue());
+    }
+    return map;
+  }
+
+  private static <K, V> Entry<K, V> e(final K key, final V value) {
+    return new Entry<K, V>() {
+
+      @Override
+      public V setValue(V value) {
+        return null;
+      }
+
+      @Override
+      public V getValue() {
+        return value;
+      }
+
+      @Override
+      public K getKey() {
+        return key;
+      }
+    };
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
----------------------------------------------------------------------
diff --git a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
index 89224f0..d183231 100644
--- a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
+++ b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
@@ -4,6 +4,7 @@ import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -32,7 +33,7 @@ public class SuperParserTest {
   private BaseFieldManager _fieldManager;
 
   @Before
-  public void setup() {
+  public void setup() throws IOException {
     // AnalyzerDefinition ad = new AnalyzerDefinition();
     // ad.setDefaultDefinition(new
     // ColumnDefinition(NoStopWordStandardAnalyzer.class.getName(), true,
@@ -52,13 +53,18 @@ public class SuperParserTest {
     parser = new SuperParser(LUCENE_VERSION, _fieldManager, true, null, ScoreType.SUPER, new Term("_primedoc_"));
   }
 
-  private BaseFieldManager getFieldManager(Analyzer a) {
+  private BaseFieldManager getFieldManager(Analyzer a) throws IOException {
     BaseFieldManager fieldManager = new BaseFieldManager(BlurConstants.SUPER, a) {
       @Override
       protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType,
           Map<String, String> props) {
         return true;
       }
+
+      @Override
+      protected void tryToLoad(String fieldName) {
+
+      }
     };
 
     fieldManager.addColumnDefinitionInt("a", "id_i");
@@ -130,7 +136,7 @@ public class SuperParserTest {
   }
 
   @Test
-  public void test5() throws ParseException {
+  public void test5() throws ParseException, IOException {
     parser = new SuperParser(LUCENE_VERSION, getFieldManager(new WhitespaceAnalyzer(LUCENE_VERSION)), true, null,
         ScoreType.SUPER, new Term("_primedoc_"));
     Query query = parser.parse("super:<a.a:a a.d:{e TO f} a.b:b a.test:hello\\<> - super:<g.c:c g.d:d>");


Mime
View raw message