incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [23/39] git commit: Final changes to get the project running again with types. All unit tests pass but I still need to add external type additions and cluster lock on adding fields via ZooKeeper so that read write contention against HDFS is lowered duri
Date Mon, 12 Aug 2013 15:49:02 GMT
Final changes to get the project running again with types.  All unit tests pass but I still need to add external type additions and cluster lock on adding fields via ZooKeeper so that read write contention against HDFS is lowered during high field creation.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2378815f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2378815f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2378815f

Branch: refs/heads/master
Commit: 2378815f1a479589095edee4d6135e776ae9b907
Parents: b708464
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Fri Aug 9 13:21:40 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Aug 9 13:21:40 2013 -0400

----------------------------------------------------------------------
 .../manager/writer/TransactionRecorder.java     |  96 ++--
 .../org/apache/blur/server/TableContext.java    |  40 +-
 .../org/apache/blur/mapreduce/BlurReducer.java  |  56 +-
 .../blur/mapreduce/lib/BlurOutputFormat.java    |  29 +-
 .../mapreduce/lib/BlurOutputFormatTest.java     |  21 +-
 .../apache/blur/analysis/BaseFieldManager.java  |  79 ++-
 .../org/apache/blur/analysis/FieldManager.java  |  75 ++-
 .../apache/blur/analysis/HdfsFieldManager.java  |  89 ++-
 .../blur/analysis/ZooKeeperFieldManager.java    | 157 +++++
 .../type/FieldLessFieldTypeDefinition.java      | 104 ++++
 .../org/apache/blur/thrift/generated/Blur.java  | 548 +++++++++---------
 .../apache/blur/thrift/generated/Metric.java    | 132 ++---
 .../blur/thrift/generated/TableDescriptor.java  | 574 ++++++++++++++++++-
 .../src/main/scripts/interface/Blur.thrift      |  21 +-
 .../main/scripts/interface/gen-html/Blur.html   |  11 +-
 .../org/apache/blur/thrift/generated/Blur.java  | 548 +++++++++---------
 .../apache/blur/thrift/generated/Metric.java    | 132 ++---
 .../blur/thrift/generated/TableDescriptor.java  | 574 ++++++++++++++++++-
 .../src/main/scripts/interface/gen-js/Blur.js   | 486 ++++++++--------
 .../main/scripts/interface/gen-js/Blur_types.js | 234 +++++---
 .../scripts/interface/gen-perl/Blur/Blur.pm     | 316 +++++-----
 .../scripts/interface/gen-perl/Blur/Types.pm    | 175 ++++--
 .../main/scripts/interface/gen-rb/blur_types.rb |  15 +-
 23 files changed, 3125 insertions(+), 1387 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
index 994c687..207d21f 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
@@ -358,30 +358,41 @@ public class TransactionRecorder extends TimerTask implements Closeable {
 
   public static List<List<Field>> getDocs(Row row, FieldManager fieldManager) throws IOException {
     List<Record> records = row.records;
+    if (records == null) {
+      return null;
+    }
     int size = records.size();
+    if (size == 0) {
+      return null;
+    }
     final String rowId = row.id;
     List<List<Field>> docs = new ArrayList<List<Field>>(size);
     for (int i = 0; i < size; i++) {
       Record record = records.get(i);
-      BlurUtil.validateRowIdAndRecord(rowId, record);
-      List<Field> fields = fieldManager.getFields(rowId, record);
-      if (i == 0) {
-        fields.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
-      }
+      List<Field> fields = getDoc(fieldManager, rowId, record);
       docs.add(fields);
     }
+    List<Field> doc = docs.get(0);
+    doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
     return docs;
   }
 
-//  public static Document convert(String rowId, Record record, Analyzer analyzer) {
-//    BlurUtil.validateRowIdAndRecord(rowId, record);
-//    Document document = new Document();
-//    document.add(new Field(BlurConstants.ROW_ID, rowId, ID_TYPE));
-//    document.add(new Field(BlurConstants.RECORD_ID, record.recordId, ID_TYPE));
-//    document.add(new Field(BlurConstants.FAMILY, record.family, ID_TYPE));
-//    addColumns(document, analyzer, record.family, record.columns);
-//    return document;
-//  }
+  public static List<Field> getDoc(FieldManager fieldManager, final String rowId, Record record) throws IOException {
+    BlurUtil.validateRowIdAndRecord(rowId, record);
+    List<Field> fields = fieldManager.getFields(rowId, record);
+    return fields;
+  }
+
+  // public static Document convert(String rowId, Record record, Analyzer
+  // analyzer) {
+  // BlurUtil.validateRowIdAndRecord(rowId, record);
+  // Document document = new Document();
+  // document.add(new Field(BlurConstants.ROW_ID, rowId, ID_TYPE));
+  // document.add(new Field(BlurConstants.RECORD_ID, record.recordId, ID_TYPE));
+  // document.add(new Field(BlurConstants.FAMILY, record.family, ID_TYPE));
+  // addColumns(document, analyzer, record.family, record.columns);
+  // return document;
+  // }
 
   private Term createRowId(String id) {
     return new Term(BlurConstants.ROW_ID, id);
@@ -409,34 +420,35 @@ public class TransactionRecorder extends TimerTask implements Closeable {
     }
   }
 
-//  public static boolean addColumns(Document document, Analyzer analyzer, String columnFamily, Iterable<Column> set) {
-//    if (set == null) {
-//      return false;
-//    }
-//    OUTER: for (Column column : set) {
-//      String name = column.getName();
-//      String value = column.value;
-//      if (value == null || name == null) {
-//        continue OUTER;
-//      }
-//      String fieldName = getFieldName(columnFamily, name);
-//      FieldType fieldType = analyzer.getFieldType(fieldName);
-//      Field field = analyzer.getField(fieldName, value, fieldType);
-//      document.add(field);
-//
-//      if (analyzer.isFullTextField(fieldName)) {
-//        document.add(new Field(SUPER, value, SUPER_FIELD_TYPE));
-//      }
-//      Set<String> subFieldNames = analyzer.getSubIndexNames(fieldName);
-//      if (subFieldNames != null) {
-//        for (String subFieldName : subFieldNames) {
-//          FieldType subFieldType = analyzer.getFieldType(subFieldName);
-//          document.add(analyzer.getField(subFieldName, value, subFieldType));
-//        }
-//      }
-//    }
-//    return true;
-//  }
+  // public static boolean addColumns(Document document, Analyzer analyzer,
+  // String columnFamily, Iterable<Column> set) {
+  // if (set == null) {
+  // return false;
+  // }
+  // OUTER: for (Column column : set) {
+  // String name = column.getName();
+  // String value = column.value;
+  // if (value == null || name == null) {
+  // continue OUTER;
+  // }
+  // String fieldName = getFieldName(columnFamily, name);
+  // FieldType fieldType = analyzer.getFieldType(fieldName);
+  // Field field = analyzer.getField(fieldName, value, fieldType);
+  // document.add(field);
+  //
+  // if (analyzer.isFullTextField(fieldName)) {
+  // document.add(new Field(SUPER, value, SUPER_FIELD_TYPE));
+  // }
+  // Set<String> subFieldNames = analyzer.getSubIndexNames(fieldName);
+  // if (subFieldNames != null) {
+  // for (String subFieldName : subFieldNames) {
+  // FieldType subFieldType = analyzer.getFieldType(subFieldName);
+  // document.add(analyzer.getField(subFieldName, value, subFieldType));
+  // }
+  // }
+  // }
+  // return true;
+  // }
 
   public static String getFieldName(String columnFamily, String name) {
     return columnFamily + SEP + name;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index c3eec1d..cc2d541 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -22,13 +22,15 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMIT
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_REFRESHS;
 import static org.apache.blur.utils.BlurConstants.SUPER;
 
+import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.blur.analysis.BaseFieldManager;
 import org.apache.blur.analysis.FieldManager;
+import org.apache.blur.analysis.HdfsFieldManager;
 import org.apache.blur.analysis.NoStopWordStandardAnalyzer;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
@@ -48,6 +50,7 @@ public class TableContext {
   private static final Log LOG = LogFactory.getLog(TableContext.class);
 
   private static final String LOGS = "logs";
+  private static final String TYPES = "types";
 
   private Path tablePath;
   private Path walTablePath;
@@ -74,11 +77,18 @@ public class TableContext {
   }
 
   public static TableContext create(TableDescriptor tableDescriptor) {
-    TableContext tableContext = cache.get(tableDescriptor.getName());
+    if (tableDescriptor == null) {
+      throw new NullPointerException("TableDescriptor can not be null.");
+    }
+    String name = tableDescriptor.getName();
+    if (name == null) {
+      throw new NullPointerException("Table name in the TableDescriptor can not be null.");
+    }
+    TableContext tableContext = cache.get(name);
     if (tableContext != null) {
       return tableContext;
     }
-    LOG.info("Creating table context for table [{0}]", tableDescriptor.getName());
+    LOG.info("Creating table context for table [{0}]", name);
     Configuration configuration = new Configuration();
     Map<String, String> tableProperties = tableDescriptor.getTableProperties();
     if (tableProperties != null) {
@@ -93,14 +103,25 @@ public class TableContext {
     tableContext.walTablePath = new Path(tableContext.tablePath, LOGS);
 
     tableContext.defaultFieldName = SUPER;
-    tableContext.table = tableDescriptor.getName();
+    tableContext.table = name;
     tableContext.descriptor = tableDescriptor;
     tableContext.timeBetweenCommits = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_COMMITS, 60000);
     tableContext.timeBetweenRefreshs = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_REFRESHS, 5000);
     tableContext.defaultPrimeDocTerm = new Term("_prime_", "true");
     tableContext.defaultScoreType = ScoreType.SUPER;
 
-    tableContext.fieldManager = null;
+    boolean strict = tableDescriptor.isStrictTypes();
+    String defaultMissingFieldType = tableDescriptor.getDefaultMissingFieldType();
+    boolean defaultMissingFieldLessIndexing = tableDescriptor.isDefaultMissingFieldLessIndexing();
+    Map<String, String> defaultMissingFieldProps = emptyIfNull(tableDescriptor.getDefaultMissingFieldProps());
+
+    Path storagePath = new Path(tableContext.tablePath, TYPES);
+    try {
+      tableContext.fieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(), storagePath,
+          configuration, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing, defaultMissingFieldProps);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
 
     Class<?> c1 = configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE,
         KeepOnlyLastCommitDeletionPolicy.class);
@@ -109,10 +130,17 @@ public class TableContext {
     Class<?> c2 = configuration.getClass(BLUR_SAHRD_INDEX_SIMILARITY, DefaultSimilarity.class);
     tableContext.similarity = (Similarity) configure(ReflectionUtils.newInstance(c2, configuration), tableContext);
 
-    cache.put(tableDescriptor.getName(), tableContext);
+    cache.put(name, tableContext);
     return tableContext;
   }
 
+  private static Map<String, String> emptyIfNull(Map<String, String> defaultMissingFieldProps) {
+    if (defaultMissingFieldProps == null) {
+      return new HashMap<String, String>();
+    }
+    return defaultMissingFieldProps;
+  }
+
   private static Object configure(Object o, TableContext tableContext) {
     if (o instanceof Configurable) {
       ((Configurable) o).setTableContext(tableContext);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
index bff087a..3669232 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
@@ -27,13 +27,12 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.search.FairSimilarity;
@@ -46,8 +45,10 @@ import org.apache.blur.mapreduce.lib.BlurOutputFormat;
 import org.apache.blur.mapreduce.lib.BlurRecord;
 import org.apache.blur.mapreduce.lib.DefaultBlurReducer;
 import org.apache.blur.mapreduce.lib.ProgressableDirectory;
+import org.apache.blur.server.TableContext;
 import org.apache.blur.store.hdfs.HdfsDirectory;
 import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
 import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurConstants;
@@ -61,6 +62,7 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
@@ -115,7 +117,7 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> {
   protected static final double MB = 1024 * 1024;
   protected IndexWriter _writer;
   protected Directory _directory;
-  protected BlurAnalyzer _analyzer;
+  protected Analyzer _analyzer;
   protected BlurTask _blurTask;
 
   protected Counter _recordCounter;
@@ -133,6 +135,7 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> {
   protected Map<String, Document> _newDocs = new HashMap<String, Document>();
   protected Set<String> _recordIdsToDelete = new HashSet<String>();
   protected Term _rowIdTerm = new Term(BlurConstants.ROW_ID);
+  private FieldManager _fieldManager;
 
   @Override
   protected void setup(Context context) throws IOException, InterruptedException {
@@ -464,43 +467,34 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> {
   }
 
   protected void setupAnalyzer(Context context) {
-    _analyzer = new BlurAnalyzer(_blurTask.getTableDescriptor().getAnalyzerDefinition());
+    TableContext tableContext = TableContext.create(_blurTask.getTableDescriptor());
+    _fieldManager = tableContext.getFieldManager();
+    _analyzer = _fieldManager.getAnalyzerForIndex();
   }
 
-  protected Document toDocument(BlurRecord record) {
+  protected Document toDocument(BlurRecord record) throws IOException {
     Document document = new Document();
     document.add(new Field(BlurConstants.ROW_ID, record.getRowId(), TransactionRecorder.ID_TYPE));
     document.add(new Field(BlurConstants.RECORD_ID, record.getRecordId(), TransactionRecorder.ID_TYPE));
 
-    String columnFamily = record.getFamily();
-    List<BlurColumn> columns = record.getColumns();
-    final Iterator<BlurColumn> iterator = columns.iterator();
-    TransactionRecorder.addColumns(document, _analyzer, columnFamily, new Iterable<Column>() {
-      @Override
-      public Iterator<Column> iterator() {
-        return new Iterator<Column>() {
-
-          @Override
-          public Column next() {
-            BlurColumn bc = iterator.next();
-            return new Column(bc.getName(), bc.getValue());
-          }
-
-          @Override
-          public boolean hasNext() {
-            return iterator.hasNext();
-          }
-
-          @Override
-          public void remove() {
-
-          }
-        };
-      }
-    });
+    List<Field> doc = TransactionRecorder.getDoc(_fieldManager, record.getRowId(), toRecord(record));
+    for (Field field : doc) {
+      document.add(field);
+    }
     return document;
   }
 
+  private Record toRecord(BlurRecord record) {
+    Record r = new Record();
+    r.setFamily(record.getFamily());
+    r.setRecordId(record.getRecordId());
+    List<BlurColumn> columns = record.getColumns();
+    for (BlurColumn blurColumn : columns) {
+      r.addToColumns(new Column(blurColumn.getName(), blurColumn.getValue()));
+    }
+    return r;
+  }
+
   protected static void report(Context context, long totalBytesCopied, long totalBytesToCopy, long startTime, String src) {
     long now = System.currentTimeMillis();
     double seconds = (now - startTime) / 1000.0;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
index ef38f48..78b0d0f 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
@@ -26,11 +26,13 @@ import java.util.Map;
 import java.util.TreeMap;
 import java.util.UUID;
 
+import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.LuceneVersionConstant;
 import org.apache.blur.manager.writer.TransactionRecorder;
 import org.apache.blur.mapreduce.lib.BlurMutate.MUTATE_TYPE;
+import org.apache.blur.server.TableContext;
 import org.apache.blur.store.hdfs.HdfsDirectory;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TJSONProtocol;
@@ -55,7 +57,9 @@ import org.apache.hadoop.mapreduce.RecordWriter;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.util.Progressable;
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.CorruptIndexException;
@@ -343,9 +347,9 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
     private static final Log LOG = LogFactory.getLog(BlurRecordWriter.class);
 
     private final Text _prevKey = new Text();
-    private final Map<String, Document> _documents = new TreeMap<String, Document>();
+    private final Map<String, List<Field>> _documents = new TreeMap<String, List<Field>>();
     private final IndexWriter _writer;
-    private final BlurAnalyzer _analyzer;
+    private final FieldManager _fieldManager;
     private final Directory _finalDir;
     private final Directory _localDir;
     private final File _localPath;
@@ -390,13 +394,16 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
       _finalDir = new ProgressableDirectory(new HdfsDirectory(configuration, _newIndex),
           BlurOutputFormat.getProgressable());
       _finalDir.setLockFactory(NoLockFactory.getNoLockFactory());
-
-      _analyzer = new BlurAnalyzer(tableDescriptor.getAnalyzerDefinition());
-      _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, _analyzer);
+      
+      TableContext tableContext = TableContext.create(tableDescriptor);
+      _fieldManager = tableContext.getFieldManager();
+      Analyzer analyzer = _fieldManager.getAnalyzerForIndex();
+      
+      _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer);
       TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy();
       mergePolicy.setUseCompoundFile(false);
 
-      _overFlowConf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, _analyzer);
+      _overFlowConf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer);
       _overFlowConf.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);
 
       if (_indexLocally) {
@@ -452,12 +459,12 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
         return;
       }
       _columnCount.increment(record.getColumns().size());
-      Document document = TransactionRecorder.convert(blurRecord.getRowId(), record, _analyzer);
-      Document dup = _documents.put(recordId, document);
+      List<Field> document = TransactionRecorder.getDoc(_fieldManager,blurRecord.getRowId(), record);
+      List<Field> dup = _documents.put(recordId, document);
       if (dup != null) {
         _recordDuplicateCount.increment(1);
       } else {
-        _fieldCount.increment(document.getFields().size());
+        _fieldCount.increment(document.size());
         _recordCount.increment(1);
       }
       flushToTmpIndexIfNeeded();
@@ -481,7 +488,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
         _localTmpWriter = new IndexWriter(_localTmpDir, _overFlowConf.clone());
         //The local tmp writer has merging disabled so the first document in is going to be doc 0.
         //Therefore the first document added is the prime doc
-        List<Document> docs = new ArrayList<Document>(_documents.values());
+        List<List<Field>> docs = new ArrayList<List<Field>>(_documents.values());
         docs.get(0).add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
         _localTmpWriter.addDocuments(docs);
       } else {
@@ -527,7 +534,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
             _rowDeleteCount.increment(1);
           }
         } else {
-          List<Document> docs = new ArrayList<Document>(_documents.values());
+          List<List<Field>> docs = new ArrayList<List<Field>>(_documents.values());
           docs.get(0).add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
           _writer.addDocuments(docs);
           _recordRateCounter.mark(_documents.size());

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
index 67135cb..1416daf 100644
--- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
+++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
@@ -30,9 +30,9 @@ import java.io.PrintWriter;
 import java.util.Collection;
 import java.util.TreeSet;
 
+import org.apache.blur.server.TableContext;
 import org.apache.blur.store.buffer.BufferStore;
 import org.apache.blur.store.hdfs.HdfsDirectory;
-import org.apache.blur.thrift.generated.AnalyzerDefinition;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.hadoop.conf.Configuration;
@@ -47,6 +47,7 @@ import org.apache.hadoop.mapreduce.TestMapReduceLocal.TrackingTextInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.lucene.index.DirectoryReader;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -61,7 +62,7 @@ public class BlurOutputFormatTest {
   private Path inDir = new Path(TEST_ROOT_DIR + "/in");
 
   @BeforeClass
-  public static void setup() throws Exception {
+  public static void setupTest() throws Exception {
     System.setProperty("test.build.data", "./target/BlurOutputFormatTest/data");
     TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "target/tmp/BlurOutputFormatTest_tmp"));
     System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log");
@@ -95,6 +96,11 @@ public class BlurOutputFormatTest {
     file.delete();
   }
 
+  @Before
+  public void setup() {
+    TableContext.clear();
+  }
+
   @Test
   public void testBlurOutputFormat() throws IOException, InterruptedException, ClassNotFoundException {
     localFs.delete(inDir, true);
@@ -113,8 +119,8 @@ public class BlurOutputFormatTest {
 
     TableDescriptor tableDescriptor = new TableDescriptor();
     tableDescriptor.setShardCount(1);
-    tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
+    tableDescriptor.setName("test");
 
     createShardDirectories(outDir, 1);
 
@@ -164,8 +170,8 @@ public class BlurOutputFormatTest {
 
     TableDescriptor tableDescriptor = new TableDescriptor();
     tableDescriptor.setShardCount(1);
-    tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
+    tableDescriptor.setName("test");
 
     createShardDirectories(outDir, 1);
 
@@ -206,8 +212,8 @@ public class BlurOutputFormatTest {
 
     TableDescriptor tableDescriptor = new TableDescriptor();
     tableDescriptor.setShardCount(2);
-    tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
+    tableDescriptor.setName("test");
 
     createShardDirectories(outDir, 2);
 
@@ -252,8 +258,8 @@ public class BlurOutputFormatTest {
 
     TableDescriptor tableDescriptor = new TableDescriptor();
     tableDescriptor.setShardCount(7);
-    tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
+    tableDescriptor.setName("test");
 
     createShardDirectories(outDir, 7);
 
@@ -299,8 +305,8 @@ public class BlurOutputFormatTest {
 
     TableDescriptor tableDescriptor = new TableDescriptor();
     tableDescriptor.setShardCount(1);
-    tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
+    tableDescriptor.setName("test");
 
     createShardDirectories(outDir, 1);
 
@@ -332,7 +338,6 @@ public class BlurOutputFormatTest {
 
     TableDescriptor tableDescriptor = new TableDescriptor();
     tableDescriptor.setShardCount(2);
-    tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
 
     createShardDirectories(outDir, 2);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
index 0404f2c..1b2ef0c 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
 import org.apache.blur.analysis.type.DoubleFieldTypeDefinition;
+import org.apache.blur.analysis.type.FieldLessFieldTypeDefinition;
 import org.apache.blur.analysis.type.FloatFieldTypeDefinition;
 import org.apache.blur.analysis.type.IntFieldTypeDefinition;
 import org.apache.blur.analysis.type.LongFieldTypeDefinition;
@@ -60,6 +61,11 @@ public abstract class BaseFieldManager extends FieldManager {
   private final Analyzer _baseAnalyzerForQuery;
   private final Analyzer _baseAnalyzerForIndex;
   private final String _fieldLessField;
+  private final Map<String, String> _defaultMissingFieldProps;
+  private final String _defaultMissingFieldType;
+  private final boolean _defaultMissingFieldLessIndexing;
+  private final boolean _strict;
+  private final FieldTypeDefinition _fieldLessFieldTypeDefinition;
 
   public static FieldType ID_TYPE;
   static {
@@ -76,7 +82,13 @@ public abstract class BaseFieldManager extends FieldManager {
     SUPER_FIELD_TYPE.setOmitNorms(true);
   }
 
-  public BaseFieldManager(String fieldLessField, final Analyzer defaultAnalyzerForQuerying) {
+  public BaseFieldManager(String fieldLessField, final Analyzer defaultAnalyzerForQuerying) throws IOException {
+    this(fieldLessField, defaultAnalyzerForQuerying, true, null, false, null);
+  }
+
+  public BaseFieldManager(String fieldLessField, final Analyzer defaultAnalyzerForQuerying, boolean strict,
+      String defaultMissingFieldType, boolean defaultMissingFieldLessIndexing,
+      Map<String, String> defaultMissingFieldProps) throws IOException {
     _typeMap.put(TextFieldTypeDefinition.NAME, TextFieldTypeDefinition.class);
     _typeMap.put(StringFieldTypeDefinition.NAME, StringFieldTypeDefinition.class);
     _typeMap.put(StoredFieldTypeDefinition.NAME, StoredFieldTypeDefinition.class);
@@ -85,6 +97,12 @@ public abstract class BaseFieldManager extends FieldManager {
     _typeMap.put(DoubleFieldTypeDefinition.NAME, DoubleFieldTypeDefinition.class);
     _typeMap.put(FloatFieldTypeDefinition.NAME, FloatFieldTypeDefinition.class);
     _fieldLessField = fieldLessField;
+    _strict = strict;
+    _defaultMissingFieldLessIndexing = defaultMissingFieldLessIndexing;
+    _defaultMissingFieldType = defaultMissingFieldType;
+    _defaultMissingFieldProps = defaultMissingFieldProps;
+
+    _fieldLessFieldTypeDefinition = new FieldLessFieldTypeDefinition();
 
     _baseAnalyzerForQuery = new AnalyzerWrapper() {
       @Override
@@ -141,11 +159,22 @@ public abstract class BaseFieldManager extends FieldManager {
       if (value == null || name == null) {
         continue;
       }
-      getAndAddFields(fields, family, column, getFieldTypeDefinition(family, column));
+      FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(family, column);
+      if (fieldTypeDefinition == null) {
+        if (isStrict()) {
+          LOG.error("Family [{0}] Column [{1}] not defined", family, column);
+          throw new IOException("Family [" + family + "] Column [" + column + "] not defined");
+        }
+        addColumnDefinition(family, name, null, getDefaultMissingFieldLessIndexing(), getDefaultMissingFieldType(),
+            getDefaultMissingFieldProps());
+        fieldTypeDefinition = getFieldTypeDefinition(family, column);
+      }
+      getAndAddFields(fields, family, column, fieldTypeDefinition);
       Collection<String> subColumns = getSubColumns(family, column);
       if (subColumns != null) {
         for (String subName : subColumns) {
-          getAndAddFields(fields, family, column, subName, getFieldTypeDefinition(family, column, subName));
+          FieldTypeDefinition subFieldTypeDefinition = getFieldTypeDefinition(family, column, subName);
+          getAndAddFields(fields, family, column, subName, subFieldTypeDefinition);
         }
       }
     }
@@ -235,6 +264,11 @@ public abstract class BaseFieldManager extends FieldManager {
     } else {
       fieldName = baseFieldName;
     }
+    return addFieldTypeDefinition(fieldName, fieldLessIndexing, fieldType, props);
+  }
+
+  private boolean addFieldTypeDefinition(String fieldName, boolean fieldLessIndexing, String fieldType,
+      Map<String, String> props) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName);
     if (fieldTypeDefinition != null) {
       return false;
@@ -277,6 +311,9 @@ public abstract class BaseFieldManager extends FieldManager {
 
   protected FieldTypeDefinition newFieldTypeDefinition(boolean fieldLessIndexing, String fieldType,
       Map<String, String> props) {
+    if (fieldType == null) {
+      throw new IllegalArgumentException("Field type can not be null.");
+    }
     Class<? extends FieldTypeDefinition> clazz = _typeMap.get(fieldType);
     if (clazz == null) {
       throw new IllegalArgumentException("FieldType of [" + fieldType + "] was not found.");
@@ -378,24 +415,36 @@ public abstract class BaseFieldManager extends FieldManager {
   @Override
   public boolean isFieldLessIndexed(String field) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
+    if (fieldTypeDefinition == null) {
+      throw new IOException("FieldTypeDefinition for field [" + field + "] is missing.");
+    }
     return fieldTypeDefinition.isFieldLessIndexed();
   }
 
   @Override
   public boolean checkSupportForFuzzyQuery(String field) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
+    if (fieldTypeDefinition == null) {
+      throw new IOException("FieldTypeDefinition for field [" + field + "] is missing.");
+    }
     return fieldTypeDefinition.checkSupportForFuzzyQuery();
   }
 
   @Override
   public boolean checkSupportForPrefixQuery(String field) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
+    if (fieldTypeDefinition == null) {
+      throw new IOException("FieldTypeDefinition for field [" + field + "] is missing.");
+    }
     return fieldTypeDefinition.checkSupportForPrefixQuery();
   }
 
   @Override
   public boolean checkSupportForWildcardQuery(String field) throws IOException {
     FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field);
+    if (fieldTypeDefinition == null) {
+      throw new IOException("FieldTypeDefinition for field [" + field + "] is missing.");
+    }
     return fieldTypeDefinition.checkSupportForWildcardQuery();
   }
 
@@ -422,6 +471,9 @@ public abstract class BaseFieldManager extends FieldManager {
 
   @Override
   public FieldTypeDefinition getFieldTypeDefinition(String field) throws IOException {
+    if (field.equals(_fieldLessField)) {
+      return _fieldLessFieldTypeDefinition;
+    }
     FieldTypeDefinition fieldTypeDefinition = _fieldNameToDefMap.get(field);
     if (fieldTypeDefinition == null) {
       tryToLoad(field);
@@ -430,8 +482,29 @@ public abstract class BaseFieldManager extends FieldManager {
     return fieldTypeDefinition;
   }
 
+  @Override
   public String getFieldLessFieldName() {
     return _fieldLessField;
   }
 
+  @Override
+  public Map<String, String> getDefaultMissingFieldProps() {
+    return _defaultMissingFieldProps;
+  }
+
+  @Override
+  public String getDefaultMissingFieldType() {
+    return _defaultMissingFieldType;
+  }
+
+  @Override
+  public boolean getDefaultMissingFieldLessIndexing() {
+    return _defaultMissingFieldLessIndexing;
+  }
+
+  @Override
+  public boolean isStrict() {
+    return _strict;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
index 2271726..519e397 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
@@ -33,7 +33,7 @@ public abstract class FieldManager {
    * 
    * @param record
    * @return
-   * @throws IOException 
+   * @throws IOException
    */
   public abstract List<Field> getFields(String rowId, Record record) throws IOException;
 
@@ -54,8 +54,8 @@ public abstract class FieldManager {
    *          the field type name, required.
    * @param props
    *          the configuration properties for this column and type.
-   * @return 
-   * @throws IOException 
+   * @return
+   * @throws IOException
    */
   public abstract boolean addColumnDefinition(String family, String columnName, String subColumnName,
       boolean fieldLessIndexing, String fieldType, Map<String, String> props) throws IOException;
@@ -66,7 +66,7 @@ public abstract class FieldManager {
    * @param fieldName
    *          the Lucene field name.
    * @return {@link Analyzer}.
-   * @throws IOException 
+   * @throws IOException
    */
   public abstract Analyzer getAnalyzerForIndex(String fieldName) throws IOException;
 
@@ -76,7 +76,7 @@ public abstract class FieldManager {
    * @param fieldName
    *          the Lucene field name.
    * @return {@link Analyzer}.
-   * @throws IOException 
+   * @throws IOException
    */
   public abstract Analyzer getAnalyzerForQuery(String fieldName) throws IOException;
 
@@ -89,7 +89,7 @@ public abstract class FieldManager {
    * @param columnName
    *          the column name.
    * @return boolean
-   * @throws IOException 
+   * @throws IOException
    */
   public abstract boolean isValidColumnDefinition(String family, String columnName) throws IOException;
 
@@ -106,7 +106,7 @@ public abstract class FieldManager {
    * @param field
    *          the field name.
    * @return boolean
-   * @throws IOException 
+   * @throws IOException
    */
   public abstract boolean checkSupportForFuzzyQuery(String field) throws IOException;
 
@@ -116,7 +116,7 @@ public abstract class FieldManager {
    * @param field
    *          the field name.
    * @return boolean
-   * @throws IOException 
+   * @throws IOException
    */
   public abstract boolean checkSupportForPrefixQuery(String field) throws IOException;
 
@@ -126,7 +126,7 @@ public abstract class FieldManager {
    * @param field
    *          the field name.
    * @return boolean
-   * @throws IOException 
+   * @throws IOException
    */
   public abstract boolean checkSupportForWildcardQuery(String field) throws IOException;
 
@@ -145,7 +145,7 @@ public abstract class FieldManager {
    * @param endInclusive
    *          if the end is inclusive.
    * @return the new range query or null.
-   * @throws IOException 
+   * @throws IOException
    */
   public abstract Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive,
       boolean endInclusive) throws IOException;
@@ -159,7 +159,7 @@ public abstract class FieldManager {
    * @param text
    *          the text for the term.
    * @return the query or null.
-   * @throws IOException 
+   * @throws IOException
    */
   public abstract Query getTermQueryIfNumeric(String field, String text) throws IOException;
 
@@ -169,14 +169,63 @@ public abstract class FieldManager {
    * @param field
    *          the field name.
    * @return the {@link FieldTypeDefinition} or null if missing.
-   * @throws IOException 
+   * @throws IOException
    */
   public abstract FieldTypeDefinition getFieldTypeDefinition(String field) throws IOException;
 
+  /**
+   * Checks to see if the field should also be indexed in the field less field.
+   * 
+   * @param name
+   *          the field name.
+   * @return boolean
+   * @throws IOException
+   */
   public abstract boolean isFieldLessIndexed(String name) throws IOException;
 
+  /**
+   * Gets the analyzer used for indexing.
+   * 
+   * @return the {@link Analyzer}.
+   */
   public abstract Analyzer getAnalyzerForIndex();
-  
+
+  /**
+   * Gets the Lucene field name of the field that is used for queries that do
+   * not specify a field.
+   * 
+   * @return the field name.
+   */
   public abstract String getFieldLessFieldName();
 
+  /**
+   * The field properties used if the table is is not in strict mode.
+   * 
+   * @return properties.
+   */
+  public abstract Map<String, String> getDefaultMissingFieldProps();
+
+  /**
+   * The field type used if the table is is not in strict mode.
+   * 
+   * @return field type.
+   */
+  public abstract String getDefaultMissingFieldType();
+
+  /**
+   * Should the field be placed in the field less indexing if the table is is
+   * not in strict mode.
+   * 
+   * @return boolean.
+   */
+  public abstract boolean getDefaultMissingFieldLessIndexing();
+
+  /**
+   * Does the table have strict field names and types. If so then if a new
+   * Column is attempted to be used for indexing then an error will be raised.
+   * 
+   * @return boolean
+   */
+  public abstract boolean isStrict();
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
index 9669e2e..9ada9f9 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
@@ -21,6 +21,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
@@ -33,10 +34,30 @@ import org.apache.lucene.analysis.Analyzer;
 
 public class HdfsFieldManager extends BaseFieldManager {
 
+  public static abstract class Lock {
+
+    public abstract void lock();
+
+    public abstract void unlock();
+
+  }
+
+  private static final Log LOG = LogFactory.getLog(HdfsFieldManager.class);
   private static final String FIELD_TYPE = "fieldType";
   private static final String FIELD_LESS_INDEXING = "fieldLessIndexing";
+  private static Lock _lock = new Lock() {
+    private final java.util.concurrent.locks.Lock _javalock = new ReentrantReadWriteLock().writeLock();
 
-  private static final Log LOG = LogFactory.getLog(HdfsFieldManager.class);
+    @Override
+    public void lock() {
+      _javalock.lock();
+    }
+
+    @Override
+    public void unlock() {
+      _javalock.unlock();
+    }
+  };
 
   private final Configuration _configuration;
   private final Path _storagePath;
@@ -44,7 +65,14 @@ public class HdfsFieldManager extends BaseFieldManager {
 
   public HdfsFieldManager(String fieldLessField, Analyzer defaultAnalyzerForQuerying, Path storagePath,
       Configuration configuration) throws IOException {
-    super(fieldLessField, defaultAnalyzerForQuerying);
+    this(fieldLessField, defaultAnalyzerForQuerying, storagePath, configuration, true, null, false, null);
+  }
+
+  public HdfsFieldManager(String fieldLessField, Analyzer defaultAnalyzerForQuerying, Path storagePath,
+      Configuration configuration, boolean strict, String defaultMissingFieldType,
+      boolean defaultMissingFieldLessIndexing, Map<String, String> defaultMissingFieldProps) throws IOException {
+    super(fieldLessField, defaultAnalyzerForQuerying, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing,
+        defaultMissingFieldProps);
     _storagePath = storagePath;
     _configuration = configuration;
     _fileSystem = _storagePath.getFileSystem(_configuration);
@@ -53,25 +81,40 @@ public class HdfsFieldManager extends BaseFieldManager {
   @Override
   protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props)
       throws IOException {
-    LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]",
-        fieldName, fieldLessIndexing, fieldType, props);
-    Properties properties = new Properties();
-    properties.setProperty(FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexing));
-    properties.setProperty(FIELD_TYPE, fieldType);
-    if (props != null) {
-      for (Entry<String, String> e : props.entrySet()) {
-        properties.put(e.getKey(), e.getValue());
+    // Might want to make this a ZK lock
+    _lock.lock();
+    try {
+      LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]",
+          fieldName, fieldLessIndexing, fieldType, props);
+      Properties properties = new Properties();
+      properties.setProperty(FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexing));
+      properties.setProperty(FIELD_TYPE, fieldType);
+      if (props != null) {
+        for (Entry<String, String> e : props.entrySet()) {
+          properties.setProperty(e.getKey(), e.getValue());
+        }
       }
+      Path path = getFieldPath(fieldName);
+      if (_fileSystem.exists(path)) {
+        LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props);
+        return false;
+      }
+      try {
+        FSDataOutputStream outputStream = _fileSystem.create(path, false);
+        properties.store(outputStream, getComments());
+        outputStream.close();
+      } catch (IOException e) {
+        if (_fileSystem.exists(path)) {
+          LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props);
+          return false;
+        } else {
+          throw e;
+        }
+      }
+      return true;
+    } finally {
+      _lock.unlock();
     }
-    Path path = getFieldPath(fieldName);
-    if (_fileSystem.exists(path)) {
-      LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props);
-      return false;
-    }
-    FSDataOutputStream outputStream = _fileSystem.create(path, false);
-    properties.store(outputStream, getComments());
-    outputStream.close();
-    return true;
   }
 
   private Path getFieldPath(String fieldName) {
@@ -105,4 +148,12 @@ public class HdfsFieldManager extends BaseFieldManager {
     }
     return result;
   }
+  
+  public static Lock getLock() {
+    return _lock;
+  }
+
+  public static void setLock(Lock lock) {
+    _lock = lock;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java
new file mode 100644
index 0000000..ccbe4f5
--- /dev/null
+++ b/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java
@@ -0,0 +1,157 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+
+public class ZooKeeperFieldManager extends BaseFieldManager {
+
+  private static final String FIELD_TYPE = "fieldType";
+  private static final String FIELD_LESS_INDEXING = "fieldLessIndexing";
+  private static final Object lock = new Object();
+
+  private static final Log LOG = LogFactory.getLog(ZooKeeperFieldManager.class);
+  private final ZooKeeper _zooKeeper;
+  private final String _basePath;
+
+  public ZooKeeperFieldManager(String fieldLessField, Analyzer defaultAnalyzerForQuerying, ZooKeeper zooKeeper, String basePath)
+      throws IOException {
+    this(fieldLessField, defaultAnalyzerForQuerying, zooKeeper, basePath, true, null, false, null);
+  }
+
+  public ZooKeeperFieldManager(String fieldLessField, Analyzer defaultAnalyzerForQuerying, ZooKeeper zooKeeper,
+      String basePath, boolean strict, String defaultMissingFieldType, boolean defaultMissingFieldLessIndexing,
+      Map<String, String> defaultMissingFieldProps) throws IOException {
+    super(fieldLessField, defaultAnalyzerForQuerying, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing,
+        defaultMissingFieldProps);
+    _zooKeeper = zooKeeper;
+    _basePath = basePath;
+  }
+
+  @Override
+  protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props)
+      throws IOException {
+    // Might want to make this a ZK lock
+    synchronized (lock) {
+      LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]",
+          fieldName, fieldLessIndexing, fieldType, props);
+      Properties properties = new Properties();
+      properties.setProperty(FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexing));
+      properties.setProperty(FIELD_TYPE, fieldType);
+      if (props != null) {
+        for (Entry<String, String> e : props.entrySet()) {
+          properties.setProperty(e.getKey(), e.getValue());
+        }
+      }
+      String path = getFieldPath(fieldName);
+      Stat stat;
+      try {
+        stat = _zooKeeper.exists(path, false);
+      } catch (KeeperException e) {
+        throw new IOException(e);
+      } catch (InterruptedException e) {
+        throw new IOException(e);
+      }
+      if (stat != null) {
+        LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props);
+        return false;
+      }
+      try {
+        _zooKeeper.create(path, toBytes(properties), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+        return true;
+      } catch (KeeperException e) {
+        if (e.code() == KeeperException.Code.NODEEXISTS) {
+          LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props);
+          return false;
+        }
+        throw new IOException(e);
+      } catch (InterruptedException e) {
+        throw new IOException(e);
+      }
+    }
+  }
+
+  private byte[] toBytes(Properties properties) throws IOException {
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    properties.store(outputStream, getComments());
+    outputStream.close();
+    return outputStream.toByteArray();
+  }
+
+  private String getFieldPath(String fieldName) {
+    return _basePath + "/" + fieldName;
+  }
+
+  private String getComments() {
+    return "This file is generated from Apache Blur to store meta data about field types.  DO NOT MODIFIY!";
+  }
+
+  @Override
+  protected void tryToLoad(String fieldName) throws IOException {
+    String path = getFieldPath(fieldName);
+    Stat stat;
+    try {
+      stat = _zooKeeper.exists(path, false);
+    } catch (KeeperException e) {
+      throw new IOException(e);
+    } catch (InterruptedException e) {
+      throw new IOException(e);
+    }
+    if (stat != null) {
+      return;
+    }
+    byte[] data;
+    try {
+      data = _zooKeeper.getData(path, false, stat);
+    } catch (KeeperException e) {
+      throw new IOException(e);
+    } catch (InterruptedException e) {
+      throw new IOException(e);
+    }
+    Properties props = new Properties();
+    ByteArrayInputStream inputStream = new ByteArrayInputStream(data);
+    props.load(inputStream);
+    inputStream.close();
+    boolean fieldLessIndexing = Boolean.parseBoolean(props.getProperty(FIELD_LESS_INDEXING));
+    String fieldType = props.getProperty(FIELD_TYPE);
+    FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldLessIndexing, fieldType, toMap(props));
+    registerFieldTypeDefinition(fieldName, fieldTypeDefinition);
+  }
+
+  private Map<String, String> toMap(Properties props) {
+    Map<String, String> result = new HashMap<String, String>();
+    for (Entry<Object, Object> e : props.entrySet()) {
+      result.put(e.getKey().toString(), e.getValue().toString());
+    }
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java
new file mode 100644
index 0000000..f6ca6f6
--- /dev/null
+++ b/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java
@@ -0,0 +1,104 @@
+package org.apache.blur.analysis.type;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.Map;
+
+import org.apache.blur.analysis.FieldTypeDefinition;
+import org.apache.blur.analysis.NoStopWordStandardAnalyzer;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+
+public class FieldLessFieldTypeDefinition extends FieldTypeDefinition {
+
+  public static final String NAME = "fieldless";
+  public static final FieldType TYPE_NOT_STORED;
+
+  static {
+    TYPE_NOT_STORED = new FieldType(TextField.TYPE_NOT_STORED);
+    TYPE_NOT_STORED.setOmitNorms(true);
+    TYPE_NOT_STORED.freeze();
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+
+  @Override
+  public void configure(Map<String, String> properties) {
+
+  }
+
+  @Override
+  public Iterable<? extends Field> getFieldsForColumn(String family, Column column) {
+    String name = getName(family, column.getName());
+    Field field = new Field(name, column.getValue(), getStoredFieldType());
+    return makeIterable(field);
+  }
+
+  @Override
+  public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, String subName) {
+    String name = getName(family, column.getName(), subName);
+    Field field = new Field(name, column.getValue(), getNotStoredFieldType());
+    return makeIterable(field);
+  }
+
+  @Override
+  public FieldType getStoredFieldType() {
+    return TYPE_NOT_STORED;
+  }
+
+  @Override
+  public FieldType getNotStoredFieldType() {
+    return TYPE_NOT_STORED;
+  }
+
+  @Override
+  public Analyzer getAnalyzerForIndex() {
+    return new NoStopWordStandardAnalyzer();
+  }
+
+  @Override
+  public Analyzer getAnalyzerForQuery() {
+    return new NoStopWordStandardAnalyzer();
+  }
+
+  @Override
+  public boolean checkSupportForFuzzyQuery() {
+    return true;
+  }
+
+  @Override
+  public boolean checkSupportForWildcardQuery() {
+    return true;
+  }
+
+  @Override
+  public boolean checkSupportForPrefixQuery() {
+    return true;
+  }
+
+  @Override
+  public boolean isNumeric() {
+    return false;
+  }
+
+}


Mime
View raw message