incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Fixed BLUR-258
Date Tue, 15 Oct 2013 13:13:12 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 261613b63 -> ea6470880


Fixed BLUR-258


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ea647088
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ea647088
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ea647088

Branch: refs/heads/apache-blur-0.2
Commit: ea6470880b241c44dbefead6c579fbea9c74dd54
Parents: 261613b
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Oct 15 09:12:21 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Oct 15 09:12:21 2013 -0400

----------------------------------------------------------------------
 .../type/CustomFieldTypeDefinition.java         |  60 +++++++-
 .../apache/blur/analysis/type/ExampleType.java  | 110 +++++++++++++++
 ...lPointVectorStrategyFieldTypeDefinition.java |  10 --
 docs/data-model.html                            | 138 +++++++++++++++++++
 4 files changed, 305 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea647088/blur-query/src/main/java/org/apache/blur/analysis/type/CustomFieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/CustomFieldTypeDefinition.java
b/blur-query/src/main/java/org/apache/blur/analysis/type/CustomFieldTypeDefinition.java
index 936541c..d098b86 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/type/CustomFieldTypeDefinition.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/type/CustomFieldTypeDefinition.java
@@ -19,45 +19,99 @@ package org.apache.blur.analysis.type;
 import org.apache.blur.analysis.FieldTypeDefinition;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.WildcardQuery;
 
 public abstract class CustomFieldTypeDefinition extends FieldTypeDefinition {
 
   private static final String NOT_SUPPORTED = "Not supported";
   private final Analyzer _queryAnalyzer = new KeywordAnalyzer();
 
+  /**
+   * The {@link #getAnalyzerForIndex(String)} should never be called for a
+   * {@link CustomFieldTypeDefinition} because this type will create the
+   * {@link IndexableField} objects from the
+   * {@link #getFieldsForColumn(String, Column)} method or
+   * {@link #getFieldsForSubColumn(String, Column, String)} method.
+   * 
+   * @throws @{@link RuntimeException}.
+   */
   @Override
-  public Analyzer getAnalyzerForIndex(String fieldName) {
+  public final Analyzer getAnalyzerForIndex(String fieldName) {
     throw new RuntimeException(NOT_SUPPORTED);
   }
 
+  /**
+   * A {@link KeywordAnalyzer} is used to parse all the queries for
+   * {@link CustomFieldTypeDefinition} types. That means that your query string
+   * must contain all the parts of your query to be passed to the
+   * {@link #getCustomQuery(String)} method.
+   * 
+   * @return {@link KeywordAnalyzer}.
+   */
   @Override
-  public Analyzer getAnalyzerForQuery(String fieldName) {
+  public final Analyzer getAnalyzerForQuery(String fieldName) {
     return _queryAnalyzer;
   }
 
+  /**
+   * Custom query types do not support {@link FuzzyQuery}.
+   * 
+   * @return false.
+   */
   @Override
   public final boolean checkSupportForFuzzyQuery() {
     return false;
   }
 
+  /**
+   * Custom query types do not support {@link WildcardQuery}.
+   * 
+   * @return false.
+   */
   @Override
   public final boolean checkSupportForWildcardQuery() {
     return false;
   }
 
+  /**
+   * Custom query types do not support {@link PrefixQuery}.
+   * 
+   * @return false.
+   */
   @Override
   public final boolean checkSupportForPrefixQuery() {
     return false;
   }
 
+  /**
+   * Checks whether this type is numeric or not. If so you need this type to be
+   * numeric please extend {@link NumericFieldTypeDefinition} instead of this
+   * class.
+   * 
+   * @return false.
+   */
   @Override
   public final boolean isNumeric() {
     return false;
   }
 
+  /**
+   * By returning true this type will create {@link Query} objects from
+   * {@link #getCustomQuery(String)} method where the entire string from the
+   * query parser is passed to the method. {@link #getCustomQuery(String)}
+   * method (true). If you want to use the {@link #getAnalyzerForIndex(String)}
+   * method to create your query, please extend {@link TextFieldTypeDefinition}
+   * or {@link FieldTypeDefinition}.
+   * 
+   * @return true.
+   */
   @Override
   public final boolean checkSupportForCustomQuery() {
     return true;
   }
-  
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea647088/blur-query/src/main/java/org/apache/blur/analysis/type/ExampleType.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/ExampleType.java b/blur-query/src/main/java/org/apache/blur/analysis/type/ExampleType.java
new file mode 100644
index 0000000..6decee8
--- /dev/null
+++ b/blur-query/src/main/java/org/apache/blur/analysis/type/ExampleType.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.analysis.type;
+
+import java.util.Map;
+
+import org.apache.blur.thrift.generated.Blur;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+
+public class ExampleType extends CustomFieldTypeDefinition {
+
+  private String _fieldNameForThisInstance;
+
+  /**
+   * Get the name of the type.
+   * 
+   * @return the name.
+   */
+  @Override
+  public String getName() {
+    return "example";
+  }
+
+  /**
+   * Configures this instance for the type.
+   * 
+   * @param fieldNameForThisInstance
+   *          the field name for this instance.
+   * @param properties
+   *          the properties passed into this type definition from the
+   *          {@link Blur.Iface#addColumnDefinition(String, org.apache.blur.thrift.generated.ColumnDefinition)}
+   *          method.
+   */
+  @Override
+  public void configure(String fieldNameForThisInstance, Map<String, String> properties,
Configuration configuration) {
+    _fieldNameForThisInstance = fieldNameForThisInstance;
+  }
+
+  /**
+   * Create {@link Field}s for the index as well as for storing the original
+   * data for retrieval.
+   * 
+   * @param family
+   *          the family name.
+   * @param column
+   *          the column that holds the name and value.
+   * 
+   * @return the {@link Iterable} of {@link Field}s.
+   */
+  @Override
+  public Iterable<? extends Field> getFieldsForColumn(String family, Column column)
{
+    String name = family + "." + column.getName();
+    String value = column.getValue();
+    return makeIterable(new StringField(name, value, Store.YES));
+  }
+
+  /**
+   * Create {@link Field}s for the index do NOT store the data because the is a
+   * sub column.
+   * 
+   * @param family
+   *          the family name.
+   * @param column
+   *          the column that holds the name and value.
+   * @param subName
+   *          the sub column name.
+   * 
+   * @return the {@link Iterable} of {@link Field}s.
+   */
+  @Override
+  public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column,
String subName) {
+    String name = family + "." + column.getName() + "." + subName;
+    String value = column.getValue();
+    return makeIterable(new StringField(name, value, Store.NO));
+  }
+
+  /**
+   * Gets the query from the text provided by the query parser.
+   * 
+   * @param text
+   *          the text provided by the query parser.
+   * @return the {@link Query}.
+   */
+  @Override
+  public Query getCustomQuery(String text) {
+    return new TermQuery(new Term(_fieldNameForThisInstance, text));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea647088/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialPointVectorStrategyFieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialPointVectorStrategyFieldTypeDefinition.java
b/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialPointVectorStrategyFieldTypeDefinition.java
index fa3eec6..ace14ee 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialPointVectorStrategyFieldTypeDefinition.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialPointVectorStrategyFieldTypeDefinition.java
@@ -21,8 +21,6 @@ import java.util.Collection;
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.spatial.query.SpatialOperation;
 import org.apache.lucene.spatial.vector.PointVectorStrategy;
 
@@ -55,12 +53,4 @@ public class SpatialPointVectorStrategyFieldTypeDefinition extends BaseSpatialFi
   public Collection<String> getAlternateFieldNames() {
     return _alternateFieldNames;
   }
-
-  @Override
-  public Analyzer getAnalyzerForIndex(String fieldName) {
-    // This happens because the doubles are tokenized but the analyzer is
-    // fetched from the Field, but the offset is still fetched from the base
-    // analyzer. This seems like a bug.
-    return new KeywordAnalyzer();
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea647088/docs/data-model.html
----------------------------------------------------------------------
diff --git a/docs/data-model.html b/docs/data-model.html
index f6c397c..8cf3a35 100644
--- a/docs/data-model.html
+++ b/docs/data-model.html
@@ -92,6 +92,19 @@
                   </li>
                 </ul>
 			  </li>
+			  <li>
+				<a href="#custom_types">Custom Types</a>
+				<ul class="nav">
+                  <li><a href="#custom_types_creating">Creating</a></li>
+                  <li><a href="#custom_types_distributing">Distributing</a></li>
+                  <li><a href="#custom_types_using">Using</a>
+				    <ul class="nav">
+                      <li><a href="#custom_types_using_cluster">&nbsp;&nbsp;Cluster
Wide</a></li>
+                      <li><a href="#custom_types_using_table">&nbsp;&nbsp;Single
Table</a></li>	
+                    </ul>
+                  </li>
+                </ul>
+              </li>
             </ul>
           </div>
         </div>
@@ -470,6 +483,131 @@ To run a query to find all the rows that contain a location within 10
miles of g
               </p>
             </div>
           </section>
+          <section>
+	         <div class="page-header">
+                <h1 id="custom_types">Custom Types</h1>
+	         </div>
+	         <p class="lead">Custom types in Blur allow you to create your own types in
Lucene
+		     as well as plugging into the query parser so that you can use your custom type.</p>
+	         <p>text here.</p>
+	         <h3 id="custom_types_creating">Creating</h3>
+	         <p>		
+You will need to extend the "org.apache.blur.analysis.FieldTypeDefinition" class found in
the blur-query module.  If you need to use a different Analyzer than the StandardAnalyzer
used in the "text" type just extend the "org.apache.blur.analysis.type.TextFieldTypeDefinition"
and make the appropriate changes.
+		     </p>
+<p>
+For types that require custom query parsing	or custom "org.apache.lucene.index.IndexableField"
manipulation without the use of an Analyzer.  Please extend "org.apache.blur.analysis.type.CustomFieldTypeDefinition".
+</p>
+<p>
+<h4>Example</h4>
+Below is a simple type that is basically the same as a "string" type, however it's implemented
by extending "org.apache.blur.analysis.type.CustomFieldTypeDefinition".<br/><br/>
+<pre><code class="java">public class ExampleType extends CustomFieldTypeDefinition
{
+
+  private String _fieldNameForThisInstance;
+
+  /**
+   * Get the name of the type.
+   * 
+   * @return the name.
+   */
+  @Override
+  public String getName() {
+    return "example";
+  }
+
+  /**
+   * Configures this instance for the type.
+   * 
+   * @param fieldNameForThisInstance
+   *          the field name for this instance.
+   * @param properties
+   *          the properties passed into this type definition from the
+   *          {@link Blur.Iface#addColumnDefinition(String, ColumnDefinition)}
+   *          method.
+   */
+  @Override
+  public void configure(String fieldNameForThisInstance, Map<String, String> properties,

+                        Configuration configuration) {
+    _fieldNameForThisInstance = fieldNameForThisInstance;
+  }
+
+  /**
+   * Create {@link Field}s for the index as well as for storing the original
+   * data for retrieval.
+   * 
+   * @param family
+   *          the family name.
+   * @param column
+   *          the column that holds the name and value.
+   * 
+   * @return the {@link Iterable} of {@link Field}s.
+   */
+  @Override
+  public Iterable&lt;? extends Field&gt; getFieldsForColumn(String family, Column
column) {
+    String name = family + "." + column.getName();
+    String value = column.getValue();
+    return makeIterable(new StringField(name, value, Store.YES));
+  }
+
+  /**
+   * Create {@link Field}s for the index do NOT store the data because the is a
+   * sub column.
+   * 
+   * @param family
+   *          the family name.
+   * @param column
+   *          the column that holds the name and value.
+   * @param subName
+   *          the sub column name.
+   * 
+   * @return the {@link Iterable} of {@link Field}s.
+   */
+  @Override
+  public Iterable&lt;? extends Field&gt; getFieldsForSubColumn(String family, Column
column, 
+       String subName) {
+    String name = family + "." + column.getName() + "." + subName;
+    String value = column.getValue();
+    return makeIterable(new StringField(name, value, Store.NO));
+  }
+
+  /**
+   * Gets the query from the text provided by the query parser.
+   * 
+   * @param text
+   *          the text provided by the query parser.
+   * @return the {@link Query}.
+   */
+  @Override
+  public Query getCustomQuery(String text) {
+    return new TermQuery(new Term(_fieldNameForThisInstance, text));
+  }
+
+}
+</code></pre>
+	
+</p>
+
+	         <h3 id="custom_types_distributing">Distributing</h3>
+	         <p>Once you have created and tested your custom type you will need to copy
the jar file containing your custom type to all the servers in the cluster.  The jar file
will need to be located within the $BLUR_HOME/lib directory.  Once there all the servers will
need to be restarted to have the jar file be picked up in the classpath.<br/><br/>In
a later version of Blur we hope to have this be a dynamic operation that can be performed
without restarting the cluster.</p>
+	         <h3 id="custom_types_using">Using</h3>
+			 <p>You can either add your custom type to the entire cluster or per table.</p>
+	         <h4 id="custom_types_using_cluster">Cluster Wide</h4>
+	         <p>For cluster wide configuration you will need to add the new field types
into the blur-site.properties file on each server.
+<pre><code class="bash">blur.fieldtype.customfield1=org.apache.blur.analysis.type.ExampleType1
+blur.fieldtype.customfield2=org.apache.blur.analysis.type.ExampleType2
+...</code></pre>
+
+Please note that the prefix of "blur.fieldtype." is all that is used from the property name
because the type gets it's name from the internal method of "getName".  However the property
names will need to be unique within the file.
+</p>
+	         <h4 id="custom_types_using_table">Single Table</h4>
+	         <p>For a single table configuration you will need to add the new field types
into the tableProperties map in the TableDescriptor as you define the table.
+<pre><code class="java">tableDescriptor.putToTableProperties("blur.fieldtype.customfield1",

+	"org.apache.blur.analysis.type.ExampleType1");
+tableDescriptor.putToTableProperties("blur.fieldtype.customfield2", 
+	"org.apache.blur.analysis.type.ExampleType2");
+...</code></pre>
+
+Please note that the prefix of "blur.fieldtype." is all that is used from the property name
because the type gets it's name from the internal method of "getName".  However the property
names will need to be unique within the map.</p>
+	      </section>
         </div>
       </div>
     </div>


Mime
View raw message