incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [4/9] git commit: Another commit to the type system rewrite.
Date Tue, 02 Jul 2013 23:17:58 GMT
Another commit to the type system rewrite.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a7a296b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a7a296b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a7a296b2

Branch: refs/heads/0.2.0-newtypesystem
Commit: a7a296b2f739fd0b1df15c153a975f5c4599c580
Parents: 8902b5f
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat Jun 29 13:26:34 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Jun 29 13:26:34 2013 -0400

----------------------------------------------------------------------
 .../analysis/AnalyzerNotFoundException.java     |  11 ++
 .../blur/analysis/FieldTypeDefinition.java      |  50 +++++++-
 .../apache/blur/analysis/LightBlurAnalyzer.java |  30 +++++
 .../blur/analysis/MemoryFieldManager.java       | 114 +++++++++++++++++++
 .../analysis/type/TextFieldTypeDefinition.java  |  73 ++++++++++++
 5 files changed, 277 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a7a296b2/blur-query/src/main/java/org/apache/blur/analysis/AnalyzerNotFoundException.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/AnalyzerNotFoundException.java
b/blur-query/src/main/java/org/apache/blur/analysis/AnalyzerNotFoundException.java
new file mode 100644
index 0000000..7d34981
--- /dev/null
+++ b/blur-query/src/main/java/org/apache/blur/analysis/AnalyzerNotFoundException.java
@@ -0,0 +1,11 @@
+package org.apache.blur.analysis;
+
+public class AnalyzerNotFoundException extends RuntimeException {
+
+  private static final long serialVersionUID = -1669681684004161773L;
+
+  public AnalyzerNotFoundException(String fieldName) {
+    super("FieldName [" + fieldName + "] not found in the analyzer");
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a7a296b2/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
index cc19cfe..7aecc95 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
@@ -16,6 +16,7 @@ package org.apache.blur.analysis;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.util.Iterator;
 import java.util.Map;
 
 import org.apache.blur.thrift.generated.Column;
@@ -47,7 +48,18 @@ public abstract class FieldTypeDefinition {
    *          the {@link Column}
    * @return the {@link Iterable} of fields.
    */
-  public abstract Iterable<? extends Field> getFields(Column column);
+  public abstract Iterable<? extends Field> getFieldsForColumn(String family, Column
column);
+
+  /**
+   * Gets the {@link Field}s for indexing from a single Column, but should not
+   * be stored because the original value should be stored in the main
+   * {@link Column}.
+   * 
+   * @param column
+   *          the {@link Column}
+   * @return the {@link Iterable} of fields.
+   */
+  public abstract Iterable<? extends Field> getFieldsForSubColumn(String family, Column
column, String subName);
 
   /**
    * Gets the {@link FieldType} for stored version. This is the normal Column
@@ -81,4 +93,40 @@ public abstract class FieldTypeDefinition {
    */
   public abstract Analyzer getAnalyzerForQuery();
 
+  protected Iterable<? extends Field> makeIterable(final Field field) {
+    return new Iterable<Field>() {
+      @Override
+      public Iterator<Field> iterator() {
+        return new Iterator<Field>() {
+
+          private boolean _hasNext = true;
+
+          @Override
+          public void remove() {
+
+          }
+
+          @Override
+          public Field next() {
+            _hasNext = false;
+            return field;
+          }
+
+          @Override
+          public boolean hasNext() {
+            return _hasNext;
+          }
+        };
+      }
+    };
+  }
+
+  protected String getName(String family, String name) {
+    return family + "." + name;
+  }
+
+  protected String getName(String family, String name, String subName) {
+    return family + "." + name + "." + subName;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a7a296b2/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java b/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java
new file mode 100644
index 0000000..a9c2c21
--- /dev/null
+++ b/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java
@@ -0,0 +1,30 @@
+package org.apache.blur.analysis;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.AnalyzerWrapper;
+
+public class LightBlurAnalyzer extends AnalyzerWrapper {
+
+  private final FieldManager _fieldManager;
+  private final boolean _usedForIndexing;
+
+  public LightBlurAnalyzer(boolean usedForIndexing, FieldManager fieldManager) {
+    _usedForIndexing = usedForIndexing;
+    _fieldManager = fieldManager;
+  }
+
+  @Override
+  protected Analyzer getWrappedAnalyzer(String fieldName) {
+    if (_usedForIndexing) {
+      return _fieldManager.getAnalyzerForIndex(fieldName);
+    } else {
+      return _fieldManager.getAnalyzerForQuery(fieldName);
+    }
+  }
+
+  @Override
+  protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents
components) {
+    return components;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a7a296b2/blur-query/src/main/java/org/apache/blur/analysis/MemoryFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/MemoryFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/MemoryFieldManager.java
new file mode 100644
index 0000000..895b748
--- /dev/null
+++ b/blur-query/src/main/java/org/apache/blur/analysis/MemoryFieldManager.java
@@ -0,0 +1,114 @@
+package org.apache.blur.analysis;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Field;
+
+public class MemoryFieldManager extends FieldManager {
+
+  private static final Log LOG = LogFactory.getLog(MemoryFieldManager.class);
+
+  private Map<String, FieldTypeDefinition> _fieldNameToDefMap = new ConcurrentHashMap<String,
FieldTypeDefinition>();
+  private Map<String, Class<? extends FieldTypeDefinition>> _typeMap = new ConcurrentHashMap<String,
Class<? extends FieldTypeDefinition>>();
+
+  @Override
+  public Iterable<? extends Field> getFields(Record record) {
+    List<Field> fields = new ArrayList<Field>();
+    String family = record.getFamily();
+    List<Column> columns = record.getColumns();
+    addDefaultFields(fields,record);
+    for (Column column : columns) {
+      getAndAddFields(fields, family, column, getFieldTypeDefinition(column));
+      for (String subName : getSubColumns(column)) {
+        getAndAddFields(fields, family, column, getFieldTypeDefinition(column, subName));
+      }
+    }
+    return fields;
+  }
+
+  private void addDefaultFields(List<Field> fields, Record record) {
+    //add family
+    //record id fields
+    
+  }
+
+  private void getAndAddFields(List<Field> fields, String family, Column column, FieldTypeDefinition
fieldTypeDefinition) {
+    for (Field field : fieldTypeDefinition.getFieldsForColumn(family, column)) {
+      fields.add(field);
+    }
+  }
+
+  private FieldTypeDefinition getFieldTypeDefinition(Column column, String subName) {
+    // TODO Auto-generated method stub
+    return null;
+  }
+
+  private List<String> getSubColumns(Column column) {
+    // TODO Auto-generated method stub
+    return null;
+  }
+
+  private FieldTypeDefinition getFieldTypeDefinition(Column column) {
+    // TODO Auto-generated method stub
+    return null;
+  }
+
+  @Override
+  public void addColumnDefinition(String family, String columnName, String subColumnName,
String fieldType,
+      Map<String, String> props) {
+    String baseFieldName = family + "." + columnName;
+    String fieldName;
+    if (subColumnName != null) {
+      if (!_fieldNameToDefMap.containsKey(baseFieldName)) {
+        throw new IllegalArgumentException("Base column of [" + baseFieldName
+            + "] not found, please add base before adding sub column.");
+      }
+      fieldName = baseFieldName + "." + subColumnName;
+    } else {
+      fieldName = baseFieldName;
+    }
+    Class<? extends FieldTypeDefinition> clazz = _typeMap.get(fieldType);
+    if (clazz == null) {
+      throw new IllegalArgumentException("FieldType of [" + fieldType + "] was not found.");
+    }
+    FieldTypeDefinition fieldTypeDefinition;
+    try {
+      fieldTypeDefinition = clazz.newInstance();
+    } catch (InstantiationException e) {
+      LOG.error("Unknown error trying to create a type of [{0}] from class [{1}]", e, fieldType,
clazz);
+      throw new RuntimeException(e);
+    } catch (IllegalAccessException e) {
+      LOG.error("Unknown error trying to create a type of [{0}] from class [{1}]", e, fieldType,
clazz);
+      throw new RuntimeException(e);
+    }
+    fieldTypeDefinition.configure(props);
+    _fieldNameToDefMap.put(fieldName, fieldTypeDefinition);
+  }
+
+  @Override
+  public Analyzer getAnalyzerForIndex(String fieldName) {
+    FieldTypeDefinition fieldTypeDefinition = _fieldNameToDefMap.get(fieldName);
+    if (fieldTypeDefinition == null) {
+      throw new AnalyzerNotFoundException(fieldName);
+    }
+    return fieldTypeDefinition.getAnalyzerForIndex();
+  }
+
+  @Override
+  public Analyzer getAnalyzerForQuery(String fieldName) {
+    FieldTypeDefinition fieldTypeDefinition = _fieldNameToDefMap.get(fieldName);
+    if (fieldTypeDefinition == null) {
+      throw new AnalyzerNotFoundException(fieldName);
+    }
+    return fieldTypeDefinition.getAnalyzerForQuery();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a7a296b2/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java
b/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java
new file mode 100644
index 0000000..7e141fe
--- /dev/null
+++ b/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java
@@ -0,0 +1,73 @@
+package org.apache.blur.analysis.type;
+
+import java.util.Map;
+
+import org.apache.blur.analysis.FieldTypeDefinition;
+import org.apache.blur.analysis.NoStopWordStandardAnalyzer;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+
+public class TextFieldTypeDefinition extends FieldTypeDefinition {
+
+  private static final String TEXT = "text";
+  private static final FieldType TYPE_NOT_STORED;
+  private static final FieldType TYPE_STORED;
+
+  static {
+    TYPE_STORED = new FieldType(TextField.TYPE_STORED);
+    TYPE_STORED.setOmitNorms(true);
+    TYPE_STORED.freeze();
+
+    TYPE_NOT_STORED = new FieldType(TextField.TYPE_NOT_STORED);
+    TYPE_NOT_STORED.setOmitNorms(true);
+    TYPE_NOT_STORED.freeze();
+  }
+
+  @Override
+  public String getName() {
+    return TEXT;
+  }
+
+  @Override
+  public void configure(Map<String, String> properties) {
+
+  }
+
+  @Override
+  public Iterable<? extends Field> getFieldsForColumn(String family, Column column)
{
+    String name = getName(family, column.getName());
+    Field field = new Field(name, column.getValue(), getStoredFieldType());
+    return makeIterable(field);
+  }
+
+  @Override
+  public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column,
String subName) {
+    String name = getName(family, column.getName(), subName);
+    Field field = new Field(name, column.getValue(), getNotStoredFieldType());
+    return makeIterable(field);
+  }
+
+  @Override
+  public FieldType getStoredFieldType() {
+    return TYPE_STORED;
+  }
+
+  @Override
+  public FieldType getNotStoredFieldType() {
+    return TYPE_NOT_STORED;
+  }
+
+  @Override
+  public Analyzer getAnalyzerForIndex() {
+    return new NoStopWordStandardAnalyzer();
+  }
+
+  @Override
+  public Analyzer getAnalyzerForQuery() {
+    return new NoStopWordStandardAnalyzer();
+  }
+
+}


Mime
View raw message