incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Adding the super indexing back in, now to fix the highlighting.
Date Wed, 19 Jun 2013 02:36:49 GMT
Updated Branches:
  refs/heads/master f04297024 -> afbe3ef35


Adding the super indexing back in, now to fix the highlighting.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/afbe3ef3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/afbe3ef3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/afbe3ef3

Branch: refs/heads/master
Commit: afbe3ef3555a80674c08a5419ff5b05b0332475e
Parents: f042970
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Jun 18 22:36:08 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Jun 18 22:36:08 2013 -0400

----------------------------------------------------------------------
 .../manager/writer/TransactionRecorder.java     | 28 ++++++++------------
 .../blur/manager/writer/IndexImporterTest.java  |  4 +--
 .../manager/writer/TransactionRecorderTest.java |  6 ++---
 .../org/apache/blur/mapreduce/BlurReducer.java  |  7 +++--
 .../blur/mapreduce/lib/BlurOutputFormat.java    |  3 +--
 5 files changed, 20 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/afbe3ef3/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
index b7453ba..8130fe7 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
@@ -17,6 +17,7 @@ package org.apache.blur.manager.writer;
  * limitations under the License.
  */
 import static org.apache.blur.utils.BlurConstants.SEP;
+import static org.apache.blur.utils.BlurConstants.SUPER;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -56,6 +57,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.BlurIndexWriter;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.Term;
@@ -356,10 +358,9 @@ public class TransactionRecorder extends TimerTask implements Closeable
{
     List<Record> records = row.records;
     int size = records.size();
     final String rowId = row.id;
-    final StringBuilder builder = new StringBuilder();
     List<Document> docs = new ArrayList<Document>(size);
     for (int i = 0; i < size; i++) {
-      Document document = convert(rowId, records.get(i), builder, analyzer);
+      Document document = convert(rowId, records.get(i), analyzer);
       if (i == 0) {
         document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE,
Store.NO));
       }
@@ -368,13 +369,13 @@ public class TransactionRecorder extends TimerTask implements Closeable
{
     return docs;
   }
 
-  public static Document convert(String rowId, Record record, StringBuilder builder, BlurAnalyzer
analyzer) {
+  public static Document convert(String rowId, Record record, BlurAnalyzer analyzer) {
     BlurUtil.validateRowIdAndRecord(rowId, record);
     Document document = new Document();
     document.add(new Field(BlurConstants.ROW_ID, rowId, ID_TYPE));
     document.add(new Field(BlurConstants.RECORD_ID, record.recordId, ID_TYPE));
     document.add(new Field(BlurConstants.FAMILY, record.family, ID_TYPE));
-    addColumns(document, analyzer, builder, record.family, record.columns);
+    addColumns(document, analyzer, record.family, record.columns);
     return document;
   }
 
@@ -403,13 +404,11 @@ public class TransactionRecorder extends TimerTask implements Closeable
{
       }
     }
   }
-  
-  public static boolean addColumns(Document document, BlurAnalyzer analyzer, StringBuilder
builder,
-      String columnFamily, Iterable<Column> set) {
+
+  public static boolean addColumns(Document document, BlurAnalyzer analyzer, String columnFamily,
Iterable<Column> set) {
     if (set == null) {
       return false;
     }
-    builder.setLength(0);
     OUTER: for (Column column : set) {
       String name = column.getName();
       String value = column.value;
@@ -420,11 +419,10 @@ public class TransactionRecorder extends TimerTask implements Closeable
{
       FieldType fieldType = analyzer.getFieldType(fieldName);
       Field field = analyzer.getField(fieldName, value, fieldType);
       document.add(field);
-      
-      // @TODO remove full text stuff
-//      if (analyzer.isFullTextField(fieldName)) {
-//        builder.append(value).append(' ');
-//      }
+
+      if (analyzer.isFullTextField(fieldName)) {
+        document.add(new Field(SUPER, value, TextField.TYPE_NOT_STORED));
+      }
       Set<String> subFieldNames = analyzer.getSubIndexNames(fieldName);
       if (subFieldNames != null) {
         for (String subFieldName : subFieldNames) {
@@ -433,10 +431,6 @@ public class TransactionRecorder extends TimerTask implements Closeable
{
         }
       }
     }
-//    if (builder.length() != 0) {
-//      String superValue = builder.toString();
-//      document.add(new Field(SUPER, superValue, Store.NO, Index.ANALYZED_NO_NORMS));
-//    }
     return true;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/afbe3ef3/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
index 5d3466e..a3e1ad9 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
@@ -113,7 +113,7 @@ public class IndexImporterTest {
   @Test
   public void testIndexImporterWithCorrectRowIdShardCombination() throws IOException {
     
-    Document document = TransactionRecorder.convert("1", genRecord("1"), new StringBuilder(),
new BlurAnalyzer());
+    Document document = TransactionRecorder.convert("1", genRecord("1"), new BlurAnalyzer());
     commitWriter.addDocument(document);
     commitWriter.commit();
     commitWriter.close();
@@ -138,7 +138,7 @@ public class IndexImporterTest {
   @Test
   public void testIndexImporterWithWrongRowIdShardCombination() throws IOException {
     setupWriter(configuration);
-    Document document = TransactionRecorder.convert("2", genRecord("1"), new StringBuilder(),
new BlurAnalyzer());
+    Document document = TransactionRecorder.convert("2", genRecord("1"), new BlurAnalyzer());
     commitWriter.addDocument(document);
     commitWriter.commit();
     commitWriter.close();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/afbe3ef3/blur-core/src/test/java/org/apache/blur/manager/writer/TransactionRecorderTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/TransactionRecorderTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/TransactionRecorderTest.java
index 17870d1..3c152da 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/TransactionRecorderTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/TransactionRecorderTest.java
@@ -136,7 +136,7 @@ public class TransactionRecorderTest {
     column.setName("columnName_123-1");
     record.setColumns(Arrays.asList(column));
     
-    TransactionRecorder.convert(rowId, record, new StringBuilder(), new BlurAnalyzer());
+    TransactionRecorder.convert(rowId, record, new BlurAnalyzer());
     assert(true);
   }
   
@@ -151,7 +151,7 @@ public class TransactionRecorderTest {
     column.setName("columnName_123-1");
     record.setColumns(Arrays.asList(column));
     
-    TransactionRecorder.convert(rowId, record, new StringBuilder(), new BlurAnalyzer());
+    TransactionRecorder.convert(rowId, record, new BlurAnalyzer());
     fail();
   }
   
@@ -166,7 +166,7 @@ public class TransactionRecorderTest {
     column.setName("columnName_123.1");
     record.setColumns(Arrays.asList(column));
     
-    TransactionRecorder.convert(rowId, record, new StringBuilder(), new BlurAnalyzer());
+    TransactionRecorder.convert(rowId, record, new BlurAnalyzer());
     fail();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/afbe3ef3/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
index f694c99..4edfb6b 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
@@ -124,7 +124,6 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate>
{
   protected Counter _fieldCounter;
   protected Counter _rowBreak;
   protected Counter _rowFailures;
-  protected StringBuilder _builder = new StringBuilder();
   protected byte[] _copyBuf;
   protected Configuration _configuration;
   protected long _start;
@@ -184,7 +183,7 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate>
{
         _recordIdsToDelete.add(record.getRecordId());
         continue;
       }
-      Document document = toDocument(record, _builder);
+      Document document = toDocument(record);
       _newDocs.put(record.getRecordId(), document);
 
       context.progress();
@@ -469,13 +468,13 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text,
BlurMutate> {
     _analyzer = new BlurAnalyzer(_blurTask.getTableDescriptor().getAnalyzerDefinition());
   }
 
-  protected Document toDocument(BlurRecord record, StringBuilder builder) {
+  protected Document toDocument(BlurRecord record) {
     Document document = new Document();
     document.add(new Field(BlurConstants.ROW_ID, record.getRowId(), TransactionRecorder.ID_TYPE));
     document.add(new Field(BlurConstants.RECORD_ID, record.getRecordId(), TransactionRecorder.ID_TYPE));
 
     String columnFamily = record.getFamily();
-    TransactionRecorder.addColumns(document, _analyzer, builder, columnFamily, new IterableConverter<BlurColumn,
Column>(
+    TransactionRecorder.addColumns(document, _analyzer, columnFamily, new IterableConverter<BlurColumn,
Column>(
         record.getColumns(), new Converter<BlurColumn, Column>() {
           @Override
           public Column convert(BlurColumn from) throws Exception {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/afbe3ef3/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
index 174c477..f8190b2 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
@@ -344,7 +344,6 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
     private final Map<String, Document> _documents = new TreeMap<String, Document>();
     private final IndexWriter _writer;
     private final BlurAnalyzer _analyzer;
-    private final StringBuilder _builder = new StringBuilder();
     private final Directory _finalDir;
     private final Directory _localDir;
     private final File _localPath;
@@ -440,7 +439,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
         _deletedRowId = blurRecord.getRowId();
         return;
       }
-      Document document = TransactionRecorder.convert(blurRecord.getRowId(), record, _builder,
_analyzer);
+      Document document = TransactionRecorder.convert(blurRecord.getRowId(), record, _analyzer);
       if (_documents.size() == 0) {
         document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE,
Store.NO));
       }


Mime
View raw message